Merge branch 'dev'

Re-structure alloc_[01](), which are mutually tail-recursive functions,
to do (unnecessary) work post-recursion so that the compiler cannot
perform tail call optimization, thus preserving intentionally unique
call paths in captured backtraces.

.gitignore

@@ -1,23 +1,72 @@
+/*.gcov.*
+
 /autom4te.cache/
+
+/bin/jemalloc.sh
+
 /config.stamp
 /config.log
 /config.status
 /configure
+
 /doc/html.xsl
 /doc/manpages.xsl
 /doc/jemalloc.xml
 /doc/jemalloc.html
 /doc/jemalloc.3
+
 /lib/
+
 /Makefile
-/include/jemalloc/internal/jemalloc_internal\.h
-/include/jemalloc/jemalloc\.h
-/include/jemalloc/jemalloc_defs\.h
-/test/jemalloc_test\.h
+
+/include/jemalloc/internal/jemalloc_internal.h
+/include/jemalloc/internal/jemalloc_internal_defs.h
+/include/jemalloc/internal/private_namespace.h
+/include/jemalloc/internal/private_unnamespace.h
+/include/jemalloc/internal/public_namespace.h
+/include/jemalloc/internal/public_symbols.txt
+/include/jemalloc/internal/public_unnamespace.h
+/include/jemalloc/internal/size_classes.h
+/include/jemalloc/jemalloc.h
+/include/jemalloc/jemalloc_defs.h
+/include/jemalloc/jemalloc_macros.h
+/include/jemalloc/jemalloc_mangle.h
+/include/jemalloc/jemalloc_mangle_jet.h
+/include/jemalloc/jemalloc_protos.h
+/include/jemalloc/jemalloc_protos_jet.h
+/include/jemalloc/jemalloc_rename.h
+
 /src/*.[od]
-/test/*.[od]
-/test/*.out
-/test/[a-z]*
-!test/*.c
-!test/*.exp
+/src/*.gcda
+/src/*.gcno
+
+/test/test.sh
+test/include/test/jemalloc_test.h
+test/include/test/jemalloc_test_defs.h
+
+/test/integration/[A-Za-z]*
+!/test/integration/[A-Za-z]*.*
+/test/integration/*.[od]
+/test/integration/*.gcda
+/test/integration/*.gcno
+/test/integration/*.out
+
+/test/src/*.[od]
+/test/src/*.gcda
+/test/src/*.gcno
+
+/test/stress/[A-Za-z]*
+!/test/stress/[A-Za-z]*.*
+/test/stress/*.[od]
+/test/stress/*.gcda
+/test/stress/*.gcno
+/test/stress/*.out
+
+/test/unit/[A-Za-z]*
+!/test/unit/[A-Za-z]*.*
+/test/unit/*.[od]
+/test/unit/*.gcda
+/test/unit/*.gcno
+/test/unit/*.out
+
 /VERSION

COPYING

@@ -1,9 +1,10 @@
 Unless otherwise specified, files in the jemalloc source distribution are
-subject to the following licenses:
+subject to the following license:
 --------------------------------------------------------------------------------
-Copyright (C) 2002-2010 Jason Evans <jasone@canonware.com>.
+Copyright (C) 2002-2014 Jason Evans <jasone@canonware.com>.
 All rights reserved.
-Copyright (C) 2007-2010 Mozilla Foundation.  All rights reserved.
+Copyright (C) 2007-2012 Mozilla Foundation.  All rights reserved.
+Copyright (C) 2009-2014 Facebook, Inc.  All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
@@ -24,28 +25,3 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 --------------------------------------------------------------------------------
-Copyright (C) 2009-2010 Facebook, Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-* Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
-* Redistributions in binary form must reproduce the above copyright notice, this
-  list of conditions and the following disclaimer in the documentation and/or
-  other materials provided with the distribution.
-* Neither the name of Facebook, Inc. nor the names of its contributors may be
-  used to endorse or promote products derived from this software without
-  specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
---------------------------------------------------------------------------------

ChangeLog

@@ -6,6 +6,277 @@ found in the git revision history:
     http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git
     git://canonware.com/jemalloc.git
 
+* 3.5.0 (January 22, 2014)
+
+  This version focuses on refactoring and automated testing, though it also
+  includes some non-trivial heap profiling optimizations not mentioned below.
+
+  New features:
+  - Add the *allocx() API, which is a successor to the experimental *allocm()
+    API.  The *allocx() functions are slightly simpler to use because they have
+    fewer parameters, they directly return the results of primary interest, and
+    mallocx()/rallocx() avoid the strict aliasing pitfall that
+    allocm()/rallocx() share with posix_memalign().  Note that *allocm() is
+    slated for removal in the next non-bugfix release.
+  - Add support for LinuxThreads.
+
+  Bug fixes:
+  - Unless heap profiling is enabled, disable floating point code and don't link
+    with libm.  This, in combination with e.g. EXTRA_CFLAGS=-mno-sse on x64
+    systems, makes it possible to completely disable floating point register
+    use.  Some versions of glibc neglect to save/restore caller-saved floating
+    point registers during dynamic lazy symbol loading, and the symbol loading
+    code uses whatever malloc the application happens to have linked/loaded
+    with, the result being potential floating point register corruption.
+  - Report ENOMEM rather than EINVAL if an OOM occurs during heap profiling
+    backtrace creation in imemalign().  This bug impacted posix_memalign() and
+    aligned_alloc().
+  - Fix a file descriptor leak in a prof_dump_maps() error path.
+  - Fix prof_dump() to close the dump file descriptor for all relevant error
+    paths.
+  - Fix rallocm() to use the arena specified by the ALLOCM_ARENA(s) flag for
+    allocation, not just deallocation.
+  - Fix a data race for large allocation stats counters.
+  - Fix a potential infinite loop during thread exit.  This bug occurred on
+    Solaris, and could affect other platforms with similar pthreads TSD
+    implementations.
+  - Don't junk-fill reallocations unless usable size changes.  This fixes a
+    violation of the *allocx()/*allocm() semantics.
+  - Fix growing large reallocation to junk fill new space.
+  - Fix huge deallocation to junk fill when munmap is disabled.
+  - Change the default private namespace prefix from empty to je_, and change
+    --with-private-namespace-prefix so that it prepends an additional prefix
+    rather than replacing je_.  This reduces the likelihood of applications
+    which statically link jemalloc experiencing symbol name collisions.
+  - Add missing private namespace mangling (relevant when
+    --with-private-namespace is specified).
+  - Add and use JEMALLOC_INLINE_C so that static inline functions are marked as
+    static even for debug builds.
+  - Add a missing mutex unlock in a malloc_init_hard() error path.  In practice
+    this error path is never executed.
+  - Fix numerous bugs in malloc_strotumax() error handling/reporting.  These
+    bugs had no impact except for malformed inputs.
+  - Fix numerous bugs in malloc_snprintf().  These bugs were not exercised by
+    existing calls, so they had no impact.
+
+* 3.4.1 (October 20, 2013)
+
+  Bug fixes:
+  - Fix a race in the "arenas.extend" mallctl that could cause memory corruption
+    of internal data structures and subsequent crashes.
+  - Fix Valgrind integration flaws that caused Valgrind warnings about reads of
+    uninitialized memory in:
+    + arena chunk headers
+    + internal zero-initialized data structures (relevant to tcache and prof
+      code)
+  - Preserve errno during the first allocation.  A readlink(2) call during
+    initialization fails unless /etc/malloc.conf exists, so errno was typically
+    set during the first allocation prior to this fix.
+  - Fix compilation warnings reported by gcc 4.8.1.
+
+* 3.4.0 (June 2, 2013)
+
+  This version is essentially a small bugfix release, but the addition of
+  aarch64 support requires that the minor version be incremented.
+
+  Bug fixes:
+  - Fix race-triggered deadlocks in chunk_record().  These deadlocks were
+    typically triggered by multiple threads concurrently deallocating huge
+    objects.
+
+  New features:
+  - Add support for the aarch64 architecture.
+
+* 3.3.1 (March 6, 2013)
+
+  This version fixes bugs that are typically encountered only when utilizing
+  custom run-time options.
+
+  Bug fixes:
+  - Fix a locking order bug that could cause deadlock during fork if heap
+    profiling were enabled.
+  - Fix a chunk recycling bug that could cause the allocator to lose track of
+    whether a chunk was zeroed.  On FreeBSD, NetBSD, and OS X, it could cause
+    corruption if allocating via sbrk(2) (unlikely unless running with the
+    "dss:primary" option specified).  This was completely harmless on Linux
+    unless using mlockall(2) (and unlikely even then, unless the
+    --disable-munmap configure option or the "dss:primary" option was
+    specified).  This regression was introduced in 3.1.0 by the
+    mlockall(2)/madvise(2) interaction fix.
+  - Fix TLS-related memory corruption that could occur during thread exit if the
+    thread never allocated memory.  Only the quarantine and prof facilities were
+    susceptible.
+  - Fix two quarantine bugs:
+    + Internal reallocation of the quarantined object array leaked the old
+      array.
+    + Reallocation failure for internal reallocation of the quarantined object
+      array (very unlikely) resulted in memory corruption.
+  - Fix Valgrind integration to annotate all internally allocated memory in a
+    way that keeps Valgrind happy about internal data structure access.
+  - Fix building for s390 systems.
+
+* 3.3.0 (January 23, 2013)
+
+  This version includes a few minor performance improvements in addition to the
+  listed new features and bug fixes.
+
+  New features:
+  - Add clipping support to lg_chunk option processing.
+  - Add the --enable-ivsalloc option.
+  - Add the --without-export option.
+  - Add the --disable-zone-allocator option.
+
+  Bug fixes:
+  - Fix "arenas.extend" mallctl to output the number of arenas.
+  - Fix chunk_recycle() to unconditionally inform Valgrind that returned memory
+    is undefined.
+  - Fix build break on FreeBSD related to alloca.h.
+
+* 3.2.0 (November 9, 2012)
+
+  In addition to a couple of bug fixes, this version modifies page run
+  allocation and dirty page purging algorithms in order to better control
+  page-level virtual memory fragmentation.
+
+  Incompatible changes:
+  - Change the "opt.lg_dirty_mult" default from 5 to 3 (32:1 to 8:1).
+
+  Bug fixes:
+  - Fix dss/mmap allocation precedence code to use recyclable mmap memory only
+    after primary dss allocation fails.
+  - Fix deadlock in the "arenas.purge" mallctl.  This regression was introduced
+    in 3.1.0 by the addition of the "arena.<i>.purge" mallctl.
+
+* 3.1.0 (October 16, 2012)
+
+  New features:
+  - Auto-detect whether running inside Valgrind, thus removing the need to
+    manually specify MALLOC_CONF=valgrind:true.
+  - Add the "arenas.extend" mallctl, which allows applications to create
+    manually managed arenas.
+  - Add the ALLOCM_ARENA() flag for {,r,d}allocm().
+  - Add the "opt.dss", "arena.<i>.dss", and "stats.arenas.<i>.dss" mallctls,
+    which provide control over dss/mmap precedence.
+  - Add the "arena.<i>.purge" mallctl, which obsoletes "arenas.purge".
+  - Define LG_QUANTUM for hppa.
+
+  Incompatible changes:
+  - Disable tcache by default if running inside Valgrind, in order to avoid
+    making unallocated objects appear reachable to Valgrind.
+  - Drop const from malloc_usable_size() argument on Linux.
+
+  Bug fixes:
+  - Fix heap profiling crash if sampled object is freed via realloc(p, 0).
+  - Remove const from __*_hook variable declarations, so that glibc can modify
+    them during process forking.
+  - Fix mlockall(2)/madvise(2) interaction.
+  - Fix fork(2)-related deadlocks.
+  - Fix error return value for "thread.tcache.enabled" mallctl.
+
+* 3.0.0 (May 11, 2012)
+
+  Although this version adds some major new features, the primary focus is on
+  internal code cleanup that facilitates maintainability and portability, most
+  of which is not reflected in the ChangeLog.  This is the first release to
+  incorporate substantial contributions from numerous other developers, and the
+  result is a more broadly useful allocator (see the git revision history for
+  contribution details).  Note that the license has been unified, thanks to
+  Facebook granting a license under the same terms as the other copyright
+  holders (see COPYING).
+
+  New features:
+  - Implement Valgrind support, redzones, and quarantine.
+  - Add support for additional platforms:
+    + FreeBSD
+    + Mac OS X Lion
+    + MinGW
+    + Windows (no support yet for replacing the system malloc)
+  - Add support for additional architectures:
+    + MIPS
+    + SH4
+    + Tilera
+  - Add support for cross compiling.
+  - Add nallocm(), which rounds a request size up to the nearest size class
+    without actually allocating.
+  - Implement aligned_alloc() (blame C11).
+  - Add the "thread.tcache.enabled" mallctl.
+  - Add the "opt.prof_final" mallctl.
+  - Update pprof (from gperftools 2.0).
+  - Add the --with-mangling option.
+  - Add the --disable-experimental option.
+  - Add the --disable-munmap option, and make it the default on Linux.
+  - Add the --enable-mremap option, which disables use of mremap(2) by default.
+
+  Incompatible changes:
+  - Enable stats by default.
+  - Enable fill by default.
+  - Disable lazy locking by default.
+  - Rename the "tcache.flush" mallctl to "thread.tcache.flush".
+  - Rename the "arenas.pagesize" mallctl to "arenas.page".
+  - Change the "opt.lg_prof_sample" default from 0 to 19 (1 B to 512 KiB).
+  - Change the "opt.prof_accum" default from true to false.
+
+  Removed features:
+  - Remove the swap feature, including the "config.swap", "swap.avail",
+    "swap.prezeroed", "swap.nfds", and "swap.fds" mallctls.
+  - Remove highruns statistics, including the
+    "stats.arenas.<i>.bins.<j>.highruns" and
+    "stats.arenas.<i>.lruns.<j>.highruns" mallctls.
+  - As part of small size class refactoring, remove the "opt.lg_[qc]space_max",
+    "arenas.cacheline", "arenas.subpage", "arenas.[tqcs]space_{min,max}", and
+    "arenas.[tqcs]bins" mallctls.
+  - Remove the "arenas.chunksize" mallctl.
+  - Remove the "opt.lg_prof_tcmax" option.
+  - Remove the "opt.lg_prof_bt_max" option.
+  - Remove the "opt.lg_tcache_gc_sweep" option.
+  - Remove the --disable-tiny option, including the "config.tiny" mallctl.
+  - Remove the --enable-dynamic-page-shift configure option.
+  - Remove the --enable-sysv configure option.
+
+  Bug fixes:
+  - Fix a statistics-related bug in the "thread.arena" mallctl that could cause
+    invalid statistics and crashes.
+  - Work around TLS deallocation via free() on Linux.  This bug could cause
+    write-after-free memory corruption.
+  - Fix a potential deadlock that could occur during interval- and
+    growth-triggered heap profile dumps.
+  - Fix large calloc() zeroing bugs due to dropping chunk map unzeroed flags.
+  - Fix chunk_alloc_dss() to stop claiming memory is zeroed.  This bug could
+    cause memory corruption and crashes with --enable-dss specified.
+  - Fix fork-related bugs that could cause deadlock in children between fork
+    and exec.
+  - Fix malloc_stats_print() to honor 'b' and 'l' in the opts parameter.
+  - Fix realloc(p, 0) to act like free(p).
+  - Do not enforce minimum alignment in memalign().
+  - Check for NULL pointer in malloc_usable_size().
+  - Fix an off-by-one heap profile statistics bug that could be observed in
+    interval- and growth-triggered heap profiles.
+  - Fix the "epoch" mallctl to update cached stats even if the passed in epoch
+    is 0.
+  - Fix bin->runcur management to fix a layout policy bug.  This bug did not
+    affect correctness.
+  - Fix a bug in choose_arena_hard() that potentially caused more arenas to be
+    initialized than necessary.
+  - Add missing "opt.lg_tcache_max" mallctl implementation.
+  - Use glibc allocator hooks to make mixed allocator usage less likely.
+  - Fix build issues for --disable-tcache.
+  - Don't mangle pthread_create() when --with-private-namespace is specified.
+
+* 2.2.5 (November 14, 2011)
+
+  Bug fixes:
+  - Fix huge_ralloc() race when using mremap(2).  This is a serious bug that
+    could cause memory corruption and/or crashes.
+  - Fix huge_ralloc() to maintain chunk statistics.
+  - Fix malloc_stats_print(..., "a") output.
+
+* 2.2.4 (November 5, 2011)
+
+  Bug fixes:
+  - Initialize arenas_tsd before using it.  This bug existed for 2.2.[0-3], as
+    well as for --disable-tls builds in earlier releases.
+  - Do not assume a 4 KiB page size in test/rallocm.c.
+
 * 2.2.3 (August 31, 2011)
 
   This version fixes numerous bugs related to heap profiling.

INSTALL

@@ -26,6 +26,19 @@ any of the following arguments (not a definitive list) to 'configure':
     Embed one or more library paths, so that libjemalloc can find the libraries
     it is linked to.  This works only on ELF-based systems.
 
+--with-mangling=<map>
+    Mangle public symbols specified in <map> which is a comma-separated list of
+    name:mangled pairs.
+
+    For example, to use ld's --wrap option as an alternative method for
+    overriding libc's malloc implementation, specify something like:
+
+      --with-mangling=malloc:__wrap_malloc,free:__wrap_free[...]
+
+    Note that mangling happens prior to application of the prefix specified by
+    --with-jemalloc-prefix, and mangled symbols are then ignored when applying
+    the prefix.
+
 --with-jemalloc-prefix=<prefix>
     Prefix all public APIs with <prefix>.  For example, if <prefix> is
     "prefix_", API changes like the following occur:
@@ -42,11 +55,16 @@ any of the following arguments (not a definitive list) to 'configure':
     jemalloc overlays the default malloc zone, but makes no attempt to actually
     replace the "malloc", "calloc", etc. symbols.
 
+--without-export
+    Don't export public APIs. This can be useful when building jemalloc as a
+    static library, or to avoid exporting public APIs when using the zone
+    allocator on OSX.
+
 --with-private-namespace=<prefix>
-    Prefix all library-private APIs with <prefix>.  For shared libraries,
+    Prefix all library-private APIs with <prefix>je_.  For shared libraries,
     symbol visibility mechanisms prevent these symbols from being exported, but
     for static libraries, naming collisions are a real possibility.  By
-    default, the prefix is "" (empty string).
+    default, <prefix> is empty, which results in a symbol prefix of je_ .
 
 --with-install-suffix=<suffix>
     Append <suffix> to the base name of all installed files, such that multiple
@@ -61,9 +79,28 @@ any of the following arguments (not a definitive list) to 'configure':
 --enable-debug
     Enable assertions and validation code.  This incurs a substantial
     performance hit, but is very useful during application development.
+    Implies --enable-ivsalloc.
 
---enable-stats
-    Enable statistics gathering functionality.  See the "opt.stats_print"
+--enable-code-coverage
+    Enable code coverage support, for use during jemalloc test development.
+    Additional testing targets are available if this option is enabled:
+
+      coverage
+      coverage_unit
+      coverage_integration
+      coverage_stress
+
+    These targets do not clear code coverage results from previous runs, and
+    there are interactions between the various coverage targets, so it is
+    usually advisable to run 'make clean' between repeated code coverage runs.
+
+--enable-ivsalloc
+    Enable validation code, which verifies that pointers reside within
+    jemalloc-owned chunks before dereferencing them. This incurs a substantial
+    performance hit.
+
+--disable-stats
+    Disable statistics gathering functionality.  See the "opt.stats_print"
     option documentation for usage details.
 
 --enable-prof
@@ -90,51 +127,54 @@ any of the following arguments (not a definitive list) to 'configure':
     Statically link against the specified libunwind.a rather than dynamically
     linking with -lunwind.
 
---disable-tiny
-    Disable tiny (sub-quantum-sized) object support.  Technically it is not
-    legal for a malloc implementation to allocate objects with less than
-    quantum alignment (8 or 16 bytes, depending on architecture), but in
-    practice it never causes any problems if, for example, 4-byte allocations
-    are 4-byte-aligned.
-
 --disable-tcache
     Disable thread-specific caches for small objects.  Objects are cached and
     released in bulk, thus reducing the total number of mutex operations.  See
     the "opt.tcache" option for usage details.
 
---enable-swap
-    Enable mmap()ed swap file support.  When this feature is built in, it is
-    possible to specify one or more files that act as backing store.  This
-    effectively allows for per application swap files.
+--enable-mremap
+    Enable huge realloc() via mremap(2).  mremap() is disabled by default
+    because the flavor used is specific to Linux, which has a quirk in its
+    virtual memory allocation algorithm that causes semi-permanent VM map holes
+    under normal jemalloc operation.
+
+--disable-munmap
+    Disable virtual memory deallocation via munmap(2); instead keep track of
+    the virtual memory for later use.  munmap() is disabled by default (i.e.
+    --disable-munmap is implied) on Linux, which has a quirk in its virtual
+    memory allocation algorithm that causes semi-permanent VM map holes under
+    normal jemalloc operation.
 
 --enable-dss
     Enable support for page allocation/deallocation via sbrk(2), in addition to
     mmap(2).
 
---enable-fill
-    Enable support for junk/zero filling of memory.  See the "opt.junk"/
-    "opt.zero" option documentation for usage details.
+--disable-fill
+    Disable support for junk/zero filling of memory, quarantine, and redzones.
+    See the "opt.junk", "opt.zero", "opt.quarantine", and "opt.redzone" option
+    documentation for usage details.
+
+--disable-valgrind
+    Disable support for Valgrind.
+
+--disable-experimental
+    Disable support for the experimental API (*allocm()).
+
+--disable-zone-allocator
+    Disable zone allocator for Darwin. This means jemalloc won't be hooked as
+    the default allocator on OSX/iOS.
+
+--enable-utrace
+    Enable utrace(2)-based allocation tracing.  This feature is not broadly
+    portable (FreeBSD has it, but Linux and OS X do not).
 
 --enable-xmalloc
     Enable support for optional immediate termination due to out-of-memory
     errors, as is commonly implemented by "xmalloc" wrapper function for malloc.
     See the "opt.xmalloc" option documentation for usage details.
 
---enable-sysv
-    Enable support for System V semantics, wherein malloc(0) returns NULL
-    rather than a minimal allocation.  See the "opt.sysv" option documentation
-    for usage details.
-
---enable-dynamic-page-shift
-    Under most conditions, the system page size never changes (usually 4KiB or
-    8KiB, depending on architecture and configuration), and unless this option
-    is enabled, jemalloc assumes that page size can safely be determined during
-    configuration and hard-coded.  Enabling dynamic page size determination has
-    a measurable impact on performance, since the compiler is forced to load
-    the page size from memory rather than embedding immediate values.
-
---disable-lazy-lock
-    Disable code that wraps pthread_create() to detect when an application
+--enable-lazy-lock
+    Enable code that wraps pthread_create() to detect when an application
     switches from single-threaded to multi-threaded mode, so that it can avoid
     mutex locking/unlocking operations while in single-threaded mode.  In
     practice, this feature usually has little impact on performance unless
@@ -181,11 +221,24 @@ PATH="?"
 
 === Advanced compilation =======================================================
 
+To build only parts of jemalloc, use the following targets:
+
+    build_lib_shared
+    build_lib_static
+    build_lib
+    build_doc_html
+    build_doc_man
+    build_doc
+
 To install only parts of jemalloc, use the following targets:
 
     install_bin
     install_include
+    install_lib_shared
+    install_lib_static
     install_lib
+    install_doc_html
+    install_doc_man
     install_doc
 
 To clean up build results to varying degrees, use the following make targets:
@@ -248,10 +301,6 @@ directory, issue configuration and build commands:
 
 The manual page is generated in both html and roff formats.  Any web browser
 can be used to view the html manual.  The roff manual page can be formatted
-prior to installation via any of the following commands:
+prior to installation via the following command:
 
     nroff -man -t doc/jemalloc.3
-
-    groff -man -t -Tps doc/jemalloc.3 | ps2pdf - doc/jemalloc.3.pdf
-
-    (cd doc; groff -man -man-ext -t -Thtml jemalloc.3 > jemalloc.3.html)

Makefile.in

@@ -17,130 +17,253 @@ INCLUDEDIR := $(DESTDIR)@INCLUDEDIR@
 LIBDIR := $(DESTDIR)@LIBDIR@
 DATADIR := $(DESTDIR)@DATADIR@
 MANDIR := $(DESTDIR)@MANDIR@
+srcroot := @srcroot@
+objroot := @objroot@
+abs_srcroot := @abs_srcroot@
+abs_objroot := @abs_objroot@
 
 # Build parameters.
-CPPFLAGS := @CPPFLAGS@ -I@srcroot@include -I@objroot@include
+CPPFLAGS := @CPPFLAGS@ -I$(srcroot)include -I$(objroot)include
 CFLAGS := @CFLAGS@
-ifeq (macho, @abi@)
-CFLAGS += -dynamic
-endif
 LDFLAGS := @LDFLAGS@
+EXTRA_LDFLAGS := @EXTRA_LDFLAGS@
 LIBS := @LIBS@
 RPATH_EXTRA := @RPATH_EXTRA@
-ifeq (macho, @abi@)
-SO := dylib
-WL_SONAME := dylib_install_name
+SO := @so@
+IMPORTLIB := @importlib@
+O := @o@
+A := @a@
+EXE := @exe@
+LIBPREFIX := @libprefix@
+REV := @rev@
+install_suffix := @install_suffix@
+ABI := @abi@
+XSLTPROC := @XSLTPROC@
+AUTOCONF := @AUTOCONF@
+_RPATH = @RPATH@
+RPATH = $(if $(1),$(call _RPATH,$(1)))
+cfghdrs_in := @cfghdrs_in@
+cfghdrs_out := @cfghdrs_out@
+cfgoutputs_in := @cfgoutputs_in@
+cfgoutputs_out := @cfgoutputs_out@
+enable_autogen := @enable_autogen@
+enable_code_coverage := @enable_code_coverage@
+enable_experimental := @enable_experimental@
+enable_zone_allocator := @enable_zone_allocator@
+DSO_LDFLAGS = @DSO_LDFLAGS@
+SOREV = @SOREV@
+PIC_CFLAGS = @PIC_CFLAGS@
+CTARGET = @CTARGET@
+LDTARGET = @LDTARGET@
+MKLIB = @MKLIB@
+AR = @AR@
+ARFLAGS = @ARFLAGS@
+CC_MM = @CC_MM@
+
+ifeq (macho, $(ABI))
+TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH="$(objroot)lib"
 else
-SO := so
-WL_SONAME := soname
-endif
-REV := 1
-ifeq (macho, @abi@)
-TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH=@objroot@lib
+ifeq (pecoff, $(ABI))
+TEST_LIBRARY_PATH := PATH="$(PATH):$(objroot)lib"
 else
 TEST_LIBRARY_PATH :=
 endif
+endif
+
+LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix)
 
 # Lists of files.
-BINS := @srcroot@bin/pprof
-CHDRS := @objroot@include/jemalloc/jemalloc@install_suffix@.h \
-	@objroot@include/jemalloc/jemalloc_defs@install_suffix@.h
-CSRCS := @srcroot@src/jemalloc.c @srcroot@src/arena.c @srcroot@src/atomic.c \
-	@srcroot@src/base.c @srcroot@src/bitmap.c @srcroot@src/chunk.c \
-	@srcroot@src/chunk_dss.c @srcroot@src/chunk_mmap.c \
-	@srcroot@src/chunk_swap.c @srcroot@src/ckh.c @srcroot@src/ctl.c \
-	@srcroot@src/extent.c @srcroot@src/hash.c @srcroot@src/huge.c \
-	@srcroot@src/mb.c @srcroot@src/mutex.c @srcroot@src/prof.c \
-	@srcroot@src/rtree.c @srcroot@src/stats.c @srcroot@src/tcache.c
-ifeq (macho, @abi@)
-CSRCS += @srcroot@src/zone.c
+BINS := $(srcroot)bin/pprof $(objroot)bin/jemalloc.sh
+C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h
+C_SRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c \
+	$(srcroot)src/atomic.c $(srcroot)src/base.c $(srcroot)src/bitmap.c \
+	$(srcroot)src/chunk.c $(srcroot)src/chunk_dss.c \
+	$(srcroot)src/chunk_mmap.c $(srcroot)src/ckh.c $(srcroot)src/ctl.c \
+	$(srcroot)src/extent.c $(srcroot)src/hash.c $(srcroot)src/huge.c \
+	$(srcroot)src/mb.c $(srcroot)src/mutex.c $(srcroot)src/prof.c \
+	$(srcroot)src/quarantine.c $(srcroot)src/rtree.c $(srcroot)src/stats.c \
+	$(srcroot)src/tcache.c $(srcroot)src/util.c $(srcroot)src/tsd.c
+ifeq ($(enable_zone_allocator), 1)
+C_SRCS += $(srcroot)src/zone.c
 endif
-STATIC_LIBS :=	@objroot@lib/libjemalloc@install_suffix@.a
-DSOS := @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) \
-	@objroot@lib/libjemalloc@install_suffix@.$(SO) \
-	@objroot@lib/libjemalloc@install_suffix@_pic.a
-MAN3 := @objroot@doc/jemalloc@install_suffix@.3
-DOCS_XML := @objroot@doc/jemalloc@install_suffix@.xml
-DOCS_HTML := $(DOCS_XML:@objroot@%.xml=@srcroot@%.html)
-DOCS_MAN3 := $(DOCS_XML:@objroot@%.xml=@srcroot@%.3)
+ifeq ($(IMPORTLIB),$(SO))
+STATIC_LIBS := $(objroot)lib/$(LIBJEMALLOC).$(A)
+endif
+ifdef PIC_CFLAGS
+STATIC_LIBS += $(objroot)lib/$(LIBJEMALLOC)_pic.$(A)
+else
+STATIC_LIBS += $(objroot)lib/$(LIBJEMALLOC)_s.$(A)
+endif
+DSOS := $(objroot)lib/$(LIBJEMALLOC).$(SOREV)
+ifneq ($(SOREV),$(SO))
+DSOS += $(objroot)lib/$(LIBJEMALLOC).$(SO)
+endif
+MAN3 := $(objroot)doc/jemalloc$(install_suffix).3
+DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml
+DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.html)
+DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.3)
 DOCS := $(DOCS_HTML) $(DOCS_MAN3)
-CTESTS := @srcroot@test/allocated.c @srcroot@test/allocm.c \
-	@srcroot@test/bitmap.c @srcroot@test/mremap.c \
-	@srcroot@test/posix_memalign.c @srcroot@test/rallocm.c \
-	@srcroot@test/thread_arena.c
+C_TESTLIB_SRCS := $(srcroot)test/src/math.c $(srcroot)test/src/mtx.c \
+	$(srcroot)test/src/SFMT.c $(srcroot)test/src/test.c \
+	$(srcroot)test/src/thd.c
+C_UTIL_INTEGRATION_SRCS := $(srcroot)src/util.c
+TESTS_UNIT := $(srcroot)test/unit/bitmap.c \
+	$(srcroot)test/unit/ckh.c \
+	$(srcroot)test/unit/hash.c \
+	$(srcroot)test/unit/junk.c \
+	$(srcroot)test/unit/mallctl.c \
+	$(srcroot)test/unit/math.c \
+	$(srcroot)test/unit/mq.c \
+	$(srcroot)test/unit/mtx.c \
+	$(srcroot)test/unit/prof_accum.c \
+	$(srcroot)test/unit/prof_gdump.c \
+	$(srcroot)test/unit/prof_idump.c \
+	$(srcroot)test/unit/ql.c \
+	$(srcroot)test/unit/qr.c \
+	$(srcroot)test/unit/quarantine.c \
+	$(srcroot)test/unit/rb.c \
+	$(srcroot)test/unit/rtree.c \
+	$(srcroot)test/unit/SFMT.c \
+	$(srcroot)test/unit/stats.c \
+	$(srcroot)test/unit/tsd.c \
+	$(srcroot)test/unit/util.c \
+	$(srcroot)test/unit/zero.c
+TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
+	$(srcroot)test/integration/allocated.c \
+	$(srcroot)test/integration/mallocx.c \
+	$(srcroot)test/integration/mremap.c \
+	$(srcroot)test/integration/posix_memalign.c \
+	$(srcroot)test/integration/rallocx.c \
+	$(srcroot)test/integration/thread_arena.c \
+	$(srcroot)test/integration/thread_tcache_enabled.c \
+	$(srcroot)test/integration/xallocx.c
+ifeq ($(enable_experimental), 1)
+TESTS_INTEGRATION += $(srcroot)test/integration/allocm.c \
+	$(srcroot)test/integration/ALLOCM_ARENA.c \
+	$(srcroot)test/integration/rallocm.c
+endif
+TESTS_STRESS :=
+TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_STRESS)
 
-.PHONY: all dist doc_html doc_man doc
+C_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.$(O))
+C_PIC_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.pic.$(O))
+C_JET_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.jet.$(O))
+C_TESTLIB_UNIT_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.unit.$(O))
+C_TESTLIB_INTEGRATION_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O))
+C_UTIL_INTEGRATION_OBJS := $(C_UTIL_INTEGRATION_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O))
+C_TESTLIB_STRESS_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.stress.$(O))
+C_TESTLIB_OBJS := $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(C_TESTLIB_STRESS_OBJS)
+
+TESTS_UNIT_OBJS := $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%.$(O))
+TESTS_INTEGRATION_OBJS := $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%.$(O))
+TESTS_STRESS_OBJS := $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%.$(O))
+TESTS_OBJS := $(TESTS_UNIT_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_STRESS_OBJS)
+
+.PHONY: all dist build_doc_html build_doc_man build_doc
 .PHONY: install_bin install_include install_lib
-.PHONY: install_html install_man install_doc install
+.PHONY: install_doc_html install_doc_man install_doc install
 .PHONY: tests check clean distclean relclean
 
-.SECONDARY : $(CTESTS:@srcroot@%.c=@objroot@%.o)
+.SECONDARY : $(TESTS_OBJS)
 
 # Default target.
-all: $(DSOS) $(STATIC_LIBS)
+all: build_lib
 
-dist: doc
+dist: build_doc
 
-@srcroot@doc/%.html : @objroot@doc/%.xml @srcroot@doc/stylesheet.xsl @objroot@doc/html.xsl
-	@XSLTPROC@ -o $@ @objroot@doc/html.xsl $<
+$(srcroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl
+	$(XSLTPROC) -o $@ $(objroot)doc/html.xsl $<
 
-@srcroot@doc/%.3 : @objroot@doc/%.xml @srcroot@doc/stylesheet.xsl @objroot@doc/manpages.xsl
-	@XSLTPROC@ -o $@ @objroot@doc/manpages.xsl $<
+$(srcroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl
+	$(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $<
 
-doc_html: $(DOCS_HTML)
-doc_man: $(DOCS_MAN3)
-doc: $(DOCS)
+build_doc_html: $(DOCS_HTML)
+build_doc_man: $(DOCS_MAN3)
+build_doc: $(DOCS)
 
 #
 # Include generated dependency files.
 #
--include $(CSRCS:@srcroot@%.c=@objroot@%.d)
--include $(CSRCS:@srcroot@%.c=@objroot@%.pic.d)
--include $(CTESTS:@srcroot@%.c=@objroot@%.d)
+ifdef CC_MM
+-include $(C_OBJS:%.$(O)=%.d)
+-include $(C_PIC_OBJS:%.$(O)=%.d)
+-include $(C_JET_OBJS:%.$(O)=%.d)
+-include $(C_TESTLIB_OBJS:%.$(O)=%.d)
+-include $(TESTS_OBJS:%.$(O)=%.d)
+endif
 
-@objroot@src/%.o: @srcroot@src/%.c
+$(C_OBJS): $(objroot)src/%.$(O): $(srcroot)src/%.c
+$(C_PIC_OBJS): $(objroot)src/%.pic.$(O): $(srcroot)src/%.c
+$(C_PIC_OBJS): CFLAGS += $(PIC_CFLAGS)
+$(C_JET_OBJS): $(objroot)src/%.jet.$(O): $(srcroot)src/%.c
+$(C_JET_OBJS): CFLAGS += -DJEMALLOC_JET
+$(C_TESTLIB_UNIT_OBJS): $(objroot)test/src/%.unit.$(O): $(srcroot)test/src/%.c
+$(C_TESTLIB_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST
+$(C_TESTLIB_INTEGRATION_OBJS): $(objroot)test/src/%.integration.$(O): $(srcroot)test/src/%.c
+$(C_TESTLIB_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST
+$(C_UTIL_INTEGRATION_OBJS): $(objroot)src/%.integration.$(O): $(srcroot)src/%.c
+$(C_TESTLIB_STRESS_OBJS): $(objroot)test/src/%.stress.$(O): $(srcroot)test/src/%.c
+$(C_TESTLIB_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST -DJEMALLOC_STRESS_TESTLIB
+$(C_TESTLIB_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include
+$(TESTS_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST
+$(TESTS_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST
+$(TESTS_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST
+$(TESTS_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c
+$(TESTS_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include
+ifneq ($(IMPORTLIB),$(SO))
+$(C_OBJS): CPPFLAGS += -DDLLEXPORT
+endif
+
+ifndef CC_MM
+# Dependencies.
+HEADER_DIRS = $(srcroot)include/jemalloc/internal \
+	$(objroot)include/jemalloc $(objroot)include/jemalloc/internal
+HEADERS = $(wildcard $(foreach dir,$(HEADER_DIRS),$(dir)/*.h))
+$(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): $(HEADERS)
+$(TESTS_OBJS): $(objroot)test/unit/jemalloc_test.h
+endif
+
+$(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): %.$(O):
 	@mkdir -p $(@D)
-	$(CC) $(CFLAGS) -c $(CPPFLAGS) -o $@ $<
-	@$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $@)))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.o \2/g\" > $(@:%.o=%.d)"
+	$(CC) $(CFLAGS) -c $(CPPFLAGS) $(CTARGET) $<
+ifdef CC_MM
+	@$(CC) -MM $(CPPFLAGS) -MT $@ -o $(@:%.$(O)=%.d) $<
+endif
 
-@objroot@src/%.pic.o: @srcroot@src/%.c
-	@mkdir -p $(@D)
-	$(CC) $(CFLAGS) -fPIC -DPIC -c $(CPPFLAGS) -o $@ $<
-	@$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $(basename $@))))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.pic.o \2/g\" > $(@:%.o=%.d)"
-
-%.$(SO) : %.$(SO).$(REV)
+ifneq ($(SOREV),$(SO))
+%.$(SO) : %.$(SOREV)
 	@mkdir -p $(@D)
 	ln -sf $(<F) $@
-
-@objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) : $(CSRCS:@srcroot@%.c=@objroot@%.pic.o)
-	@mkdir -p $(@D)
-	$(CC) -shared -Wl,-$(WL_SONAME),$(@F) $(RPATH_EXTRA:%=@RPATH@%) -o $@ $+ $(LDFLAGS) $(LIBS)
-
-@objroot@lib/libjemalloc@install_suffix@_pic.a : $(CSRCS:@srcroot@%.c=@objroot@%.pic.o)
-	@mkdir -p $(@D)
-	ar crus $@ $+
-
-@objroot@lib/libjemalloc@install_suffix@.a : $(CSRCS:@srcroot@%.c=@objroot@%.o)
-	@mkdir -p $(@D)
-	ar crus $@ $+
-
-@objroot@test/%.o: @srcroot@test/%.c
-	@mkdir -p $(@D)
-	$(CC) $(CFLAGS) -c $(CPPFLAGS) -I@objroot@test -o $@ $<
-	@$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) -I@objroot@test $< | sed \"s/\($(subst /,\/,$(notdir $(basename $@)))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.o \2/g\" > $(@:%.o=%.d)"
-
-# Automatic dependency generation misses #include "*.c".
-@objroot@test/bitmap.o : @objroot@src/bitmap.o
-
-@objroot@test/%: @objroot@test/%.o \
-		 @objroot@lib/libjemalloc@install_suffix@.$(SO)
-	@mkdir -p $(@D)
-ifneq (@RPATH@, )
-	$(CC) -o $@ $< @RPATH@@objroot@lib -L@objroot@lib -ljemalloc@install_suffix@ -lpthread
-else
-	$(CC) -o $@ $< -L@objroot@lib -ljemalloc@install_suffix@ -lpthread
 endif
 
+$(objroot)lib/$(LIBJEMALLOC).$(SOREV) : $(if $(PIC_CFLAGS),$(C_PIC_OBJS),$(C_OBJS))
+	@mkdir -p $(@D)
+	$(CC) $(DSO_LDFLAGS) $(call RPATH,$(RPATH_EXTRA)) $(LDTARGET) $+ $(LDFLAGS) $(LIBS) $(EXTRA_LDFLAGS)
+
+$(objroot)lib/$(LIBJEMALLOC)_pic.$(A) : $(C_PIC_OBJS)
+$(objroot)lib/$(LIBJEMALLOC).$(A) : $(C_OBJS)
+$(objroot)lib/$(LIBJEMALLOC)_s.$(A) : $(C_OBJS)
+
+$(STATIC_LIBS):
+	@mkdir -p $(@D)
+	$(AR) $(ARFLAGS)@AROUT@ $+
+
+$(objroot)test/unit/%$(EXE): $(objroot)test/unit/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_UNIT_OBJS)
+	@mkdir -p $(@D)
+	$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS)
+
+$(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
+	@mkdir -p $(@D)
+	$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(filter -lpthread,$(LIBS))) -lm $(EXTRA_LDFLAGS)
+
+$(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_STRESS_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
+	@mkdir -p $(@D)
+	$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS)
+
+build_lib_shared: $(DSOS)
+build_lib_static: $(STATIC_LIBS)
+build_lib: build_lib_shared build_lib_static
+
 install_bin:
 	install -d $(BINDIR)
 	@for b in $(BINS); do \
@@ -150,110 +273,157 @@ done
 
 install_include:
 	install -d $(INCLUDEDIR)/jemalloc
-	@for h in $(CHDRS); do \
+	@for h in $(C_HDRS); do \
 	echo "install -m 644 $$h $(INCLUDEDIR)/jemalloc"; \
 	install -m 644 $$h $(INCLUDEDIR)/jemalloc; \
 done
 
-install_lib: $(DSOS) $(STATIC_LIBS)
+install_lib_shared: $(DSOS)
 	install -d $(LIBDIR)
-	install -m 755 @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR)
-	ln -sf libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR)/libjemalloc@install_suffix@.$(SO)
-	install -m 755 @objroot@lib/libjemalloc@install_suffix@_pic.a $(LIBDIR)
-	install -m 755 @objroot@lib/libjemalloc@install_suffix@.a $(LIBDIR)
+	install -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR)
+ifneq ($(SOREV),$(SO))
+	ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO)
+endif
 
-install_html:
-	install -d $(DATADIR)/doc/jemalloc@install_suffix@
-	@for d in $(DOCS_HTML); do \
-	echo "install -m 644 $$d $(DATADIR)/doc/jemalloc@install_suffix@"; \
-	install -m 644 $$d $(DATADIR)/doc/jemalloc@install_suffix@; \
+install_lib_static: $(STATIC_LIBS)
+	install -d $(LIBDIR)
+	@for l in $(STATIC_LIBS); do \
+	echo "install -m 755 $$l $(LIBDIR)"; \
+	install -m 755 $$l $(LIBDIR); \
 done
 
-install_man:
+install_lib: install_lib_shared install_lib_static
+
+install_doc_html:
+	install -d $(DATADIR)/doc/jemalloc$(install_suffix)
+	@for d in $(DOCS_HTML); do \
+	echo "install -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \
+	install -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \
+done
+
+install_doc_man:
 	install -d $(MANDIR)/man3
 	@for d in $(DOCS_MAN3); do \
 	echo "install -m 644 $$d $(MANDIR)/man3"; \
 	install -m 644 $$d $(MANDIR)/man3; \
 done
 
-install_doc: install_html install_man
+install_doc: install_doc_html install_doc_man
 
 install: install_bin install_include install_lib install_doc
 
-tests: $(CTESTS:@srcroot@%.c=@objroot@%)
+tests_unit: $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%$(EXE))
+tests_integration: $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%$(EXE))
+tests_stress: $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%$(EXE))
+tests: tests_unit tests_integration tests_stress
 
-check: tests
-	@mkdir -p @objroot@test
-	@$(SHELL) -c 'total=0; \
-		failures=0; \
-		echo "========================================="; \
-		for t in $(CTESTS:@srcroot@%.c=@objroot@%); do \
-			total=`expr $$total + 1`; \
-			/bin/echo -n "$${t} ... "; \
-			$(TEST_LIBRARY_PATH) $${t} @abs_srcroot@ @abs_objroot@ \
-			  > @objroot@$${t}.out 2>&1; \
-			if test -e "@srcroot@$${t}.exp"; then \
-				diff -u @srcroot@$${t}.exp \
-				  @objroot@$${t}.out >/dev/null 2>&1; \
-				fail=$$?; \
-				if test "$${fail}" -eq "1" ; then \
-					failures=`expr $${failures} + 1`; \
-					echo "*** FAIL ***"; \
-				else \
-					echo "pass"; \
-				fi; \
-			else \
-				echo "*** FAIL *** (.exp file is missing)"; \
-				failures=`expr $${failures} + 1`; \
-			fi; \
-		done; \
-		echo "========================================="; \
-		echo "Failures: $${failures}/$${total}"'
+check_unit_dir:
+	@mkdir -p $(objroot)test/unit
+check_integration_dir:
+	@mkdir -p $(objroot)test/integration
+check_stress_dir:
+	@mkdir -p $(objroot)test/stress
+check_dir: check_unit_dir check_integration_dir check_stress_dir
+
+check_unit: tests_unit check_unit_dir
+	$(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%)
+check_integration: tests_integration check_integration_dir
+	$(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
+check_stress: tests_stress check_stress_dir
+	$(SHELL) $(objroot)test/test.sh $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%)
+check: tests check_dir
+	$(SHELL) $(objroot)test/test.sh $(TESTS:$(srcroot)%.c=$(objroot)%)
+
+ifeq ($(enable_code_coverage), 1)
+coverage_unit: check_unit
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src unit $(C_TESTLIB_UNIT_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/unit unit $(TESTS_UNIT_OBJS)
+
+coverage_integration: check_integration
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src integration $(C_UTIL_INTEGRATION_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS)
+
+coverage_stress: check_stress
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/stress stress $(TESTS_STRESS_OBJS)
+
+coverage: check
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src integration $(C_UTIL_INTEGRATION_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src unit $(C_TESTLIB_UNIT_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/unit unit $(TESTS_UNIT_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/stress integration $(TESTS_STRESS_OBJS)
+endif
 
 clean:
-	rm -f $(CSRCS:@srcroot@%.c=@objroot@%.o)
-	rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.o)
-	rm -f $(CSRCS:@srcroot@%.c=@objroot@%.d)
-	rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.d)
-	rm -f $(CTESTS:@srcroot@%.c=@objroot@%)
-	rm -f $(CTESTS:@srcroot@%.c=@objroot@%.o)
-	rm -f $(CTESTS:@srcroot@%.c=@objroot@%.d)
-	rm -f $(CTESTS:@srcroot@%.c=@objroot@%.out)
+	rm -f $(C_OBJS)
+	rm -f $(C_PIC_OBJS)
+	rm -f $(C_JET_OBJS)
+	rm -f $(C_TESTLIB_OBJS)
+	rm -f $(C_OBJS:%.$(O)=%.d)
+	rm -f $(C_OBJS:%.$(O)=%.gcda)
+	rm -f $(C_OBJS:%.$(O)=%.gcno)
+	rm -f $(C_PIC_OBJS:%.$(O)=%.d)
+	rm -f $(C_PIC_OBJS:%.$(O)=%.gcda)
+	rm -f $(C_PIC_OBJS:%.$(O)=%.gcno)
+	rm -f $(C_JET_OBJS:%.$(O)=%.d)
+	rm -f $(C_JET_OBJS:%.$(O)=%.gcda)
+	rm -f $(C_JET_OBJS:%.$(O)=%.gcno)
+	rm -f $(C_TESTLIB_OBJS:%.$(O)=%.d)
+	rm -f $(C_TESTLIB_OBJS:%.$(O)=%.gcda)
+	rm -f $(C_TESTLIB_OBJS:%.$(O)=%.gcno)
+	rm -f $(TESTS_OBJS:%.$(O)=%$(EXE))
+	rm -f $(TESTS_OBJS)
+	rm -f $(TESTS_OBJS:%.$(O)=%.d)
+	rm -f $(TESTS_OBJS:%.$(O)=%.gcda)
+	rm -f $(TESTS_OBJS:%.$(O)=%.gcno)
+	rm -f $(TESTS_OBJS:%.$(O)=%.out)
 	rm -f $(DSOS) $(STATIC_LIBS)
+	rm -f $(objroot)*.gcov.*
 
 distclean: clean
-	rm -rf @objroot@autom4te.cache
-	rm -f @objroot@config.log
-	rm -f @objroot@config.status
-	rm -f @objroot@config.stamp
-	rm -f @cfghdrs_out@
-	rm -f @cfgoutputs_out@
+	rm -rf $(objroot)autom4te.cache
+	rm -f $(objroot)bin/jemalloc.sh
+	rm -f $(objroot)config.log
+	rm -f $(objroot)config.status
+	rm -f $(objroot)config.stamp
+	rm -f $(cfghdrs_out)
+	rm -f $(cfgoutputs_out)
 
 relclean: distclean
-	rm -f @objroot@configure
-	rm -f @srcroot@VERSION
+	rm -f $(objroot)configure
+	rm -f $(srcroot)VERSION
 	rm -f $(DOCS_HTML)
 	rm -f $(DOCS_MAN3)
 
 #===============================================================================
 # Re-configuration rules.
 
-ifeq (@enable_autogen@, 1)
-@srcroot@configure : @srcroot@configure.ac
-	cd ./@srcroot@ && @AUTOCONF@
+ifeq ($(enable_autogen), 1)
+$(srcroot)configure : $(srcroot)configure.ac
+	cd ./$(srcroot) && $(AUTOCONF)
 
-@objroot@config.status : @srcroot@configure
-	./@objroot@config.status --recheck
+$(objroot)config.status : $(srcroot)configure
+	./$(objroot)config.status --recheck
 
-@srcroot@config.stamp.in : @srcroot@configure.ac
-	echo stamp > @srcroot@config.stamp.in
+$(srcroot)config.stamp.in : $(srcroot)configure.ac
+	echo stamp > $(srcroot)config.stamp.in
 
-@objroot@config.stamp : @cfgoutputs_in@ @cfghdrs_in@ @srcroot@configure
-	./@objroot@config.status
+$(objroot)config.stamp : $(cfgoutputs_in) $(cfghdrs_in) $(srcroot)configure
+	./$(objroot)config.status
 	@touch $@
 
 # There must be some action in order for make to re-read Makefile when it is
 # out of date.
-@cfgoutputs_out@ @cfghdrs_out@ : @objroot@config.stamp
+$(cfgoutputs_out) $(cfghdrs_out) : $(objroot)config.stamp
 	@true
 endif

README

@@ -1,10 +1,14 @@
-jemalloc is a general-purpose scalable concurrent malloc(3) implementation.
-This distribution is a stand-alone "portable" implementation that currently
-targets Linux and Apple OS X.  jemalloc is included as the default allocator in
-the FreeBSD and NetBSD operating systems, and it is used by the Mozilla Firefox
-web browser on Microsoft Windows-related platforms.  Depending on your needs,
-one of the other divergent versions may suit your needs better than this
-distribution.
+jemalloc is a general purpose malloc(3) implementation that emphasizes
+fragmentation avoidance and scalable concurrency support.  jemalloc first came
+into use as the FreeBSD libc allocator in 2005, and since then it has found its
+way into numerous applications that rely on its predictable behavior.  In 2010
+jemalloc development efforts broadened to include developer support features
+such as heap profiling, Valgrind integration, and extensive monitoring/tuning
+hooks.  Modern jemalloc releases continue to be integrated back into FreeBSD,
+and therefore versatility remains critical.  Ongoing development efforts trend
+toward making jemalloc among the best allocators for a broad range of demanding
+applications, and eliminating/mitigating weaknesses that have practical
+repercussions for real world applications.
 
 The COPYING file contains copyright and licensing information.
 

bin/jemalloc.sh.in

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+
+@LD_PRELOAD_VAR@=${libdir}/libjemalloc.@SOREV@
+export @LD_PRELOAD_VAR@
+exec "$@"

config.sub

@@ -1,42 +1,40 @@
 #! /bin/sh
 # Configuration validation subroutine script.
-#   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
-#   2000, 2001, 2002, 2003 Free Software Foundation, Inc.
+#   Copyright 1992-2013 Free Software Foundation, Inc.
 
-timestamp='2004-02-23'
+timestamp='2013-10-01'
 
-# This file is (in principle) common to ALL GNU software.
-# The presence of a machine in this file suggests that SOME GNU software
-# can handle that machine.  It does not imply ALL GNU software can.
-#
-# This file is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+#
 # As a special exception to the GNU General Public License, if you
 # distribute this file as part of a program that contains a
 # configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
+# the same distribution terms that you use for the rest of that
+# program.  This Exception is an additional permission under section 7
+# of the GNU General Public License, version 3 ("GPLv3").
 
-# Please send patches to <config-patches@gnu.org>.  Submit a context
-# diff and a properly formatted ChangeLog entry.
+
+# Please send patches with a ChangeLog entry to config-patches@gnu.org.
 #
 # Configuration subroutine to validate and canonicalize a configuration type.
 # Supply the specified configuration type as an argument.
 # If it is invalid, we print an error message on stderr and exit with code 1.
 # Otherwise, we print the canonical config type on stdout and succeed.
 
+# You can get the latest version of this script from:
+# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
+
 # This file is supposed to be the same for all GNU packages
 # and recognize all the CPU types, system types and aliases
 # that are meaningful with *any* GNU software.
@@ -70,8 +68,7 @@ Report bugs and patches to <config-patches@gnu.org>."
 version="\
 GNU config.sub ($timestamp)
 
-Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001
-Free Software Foundation, Inc.
+Copyright 1992-2013 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -83,11 +80,11 @@ Try \`$me --help' for more information."
 while test $# -gt 0 ; do
   case $1 in
     --time-stamp | --time* | -t )
-       echo "$timestamp" ; exit 0 ;;
+       echo "$timestamp" ; exit ;;
     --version | -v )
-       echo "$version" ; exit 0 ;;
+       echo "$version" ; exit ;;
     --help | --h* | -h )
-       echo "$usage"; exit 0 ;;
+       echo "$usage"; exit ;;
     -- )     # Stop option processing
        shift; break ;;
     - )	# Use stdin as input.
@@ -99,7 +96,7 @@ while test $# -gt 0 ; do
     *local*)
        # First pass through any local machine types.
        echo $1
-       exit 0;;
+       exit ;;
 
     * )
        break ;;
@@ -118,11 +115,18 @@ esac
 # Here we must recognize all the valid KERNEL-OS combinations.
 maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
 case $maybe_os in
-  nto-qnx* | linux-gnu* | linux-dietlibc | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | \
-  kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | storm-chaos* | os2-emx* | rtmk-nova*)
+  nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \
+  linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
+  knetbsd*-gnu* | netbsd*-gnu* | \
+  kopensolaris*-gnu* | \
+  storm-chaos* | os2-emx* | rtmk-nova*)
     os=-$maybe_os
     basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
     ;;
+  android-linux)
+    os=-linux-android
+    basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown
+    ;;
   *)
     basic_machine=`echo $1 | sed 's/-[^-]*$//'`
     if [ $basic_machine != $1 ]
@@ -145,10 +149,13 @@ case $os in
 	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
 	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
 	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \
-	-apple | -axis)
+	-apple | -axis | -knuth | -cray | -microblaze*)
 		os=
 		basic_machine=$1
 		;;
+	-bluegene*)
+		os=-cnk
+		;;
 	-sim | -cisco | -oki | -wec | -winbond)
 		os=
 		basic_machine=$1
@@ -163,13 +170,17 @@ case $os in
 		os=-chorusos
 		basic_machine=$1
 		;;
- 	-chorusrdb)
- 		os=-chorusrdb
+	-chorusrdb)
+		os=-chorusrdb
 		basic_machine=$1
- 		;;
+		;;
 	-hiux*)
 		os=-hiuxwe2
 		;;
+	-sco6)
+		os=-sco5v6
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
 	-sco5)
 		os=-sco3.2v5
 		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
@@ -186,6 +197,10 @@ case $os in
 		# Don't forget version if it is 3.2v4 or newer.
 		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
 		;;
+	-sco5v6*)
+		# Don't forget version if it is 3.2v4 or newer.
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
 	-sco*)
 		os=-sco3.2v2
 		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
@@ -203,6 +218,12 @@ case $os in
 	-isc*)
 		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
 		;;
+	-lynx*178)
+		os=-lynxos178
+		;;
+	-lynx*5)
+		os=-lynxos5
+		;;
 	-lynx*)
 		os=-lynxos
 		;;
@@ -227,57 +248,107 @@ case $basic_machine in
 	# Some are omitted here because they have special meanings below.
 	1750a | 580 \
 	| a29k \
+	| aarch64 | aarch64_be \
 	| alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \
 	| alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \
 	| am33_2.0 \
-	| arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr \
-	| c4x | clipper \
+	| arc | arceb \
+	| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \
+	| avr | avr32 \
+	| be32 | be64 \
+	| bfin \
+	| c4x | c8051 | clipper \
 	| d10v | d30v | dlx | dsp16xx \
-	| fr30 | frv \
+	| epiphany \
+	| fido | fr30 | frv \
 	| h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
+	| hexagon \
 	| i370 | i860 | i960 | ia64 \
 	| ip2k | iq2000 \
-	| m32r | m68000 | m68k | m88k | mcore \
+	| k1om \
+	| le32 | le64 \
+	| lm32 \
+	| m32c | m32r | m32rle | m68000 | m68k | m88k \
+	| maxq | mb | microblaze | microblazeel | mcore | mep | metag \
 	| mips | mipsbe | mipseb | mipsel | mipsle \
 	| mips16 \
 	| mips64 | mips64el \
-	| mips64vr | mips64vrel \
+	| mips64octeon | mips64octeonel \
 	| mips64orion | mips64orionel \
+	| mips64r5900 | mips64r5900el \
+	| mips64vr | mips64vrel \
 	| mips64vr4100 | mips64vr4100el \
 	| mips64vr4300 | mips64vr4300el \
 	| mips64vr5000 | mips64vr5000el \
+	| mips64vr5900 | mips64vr5900el \
 	| mipsisa32 | mipsisa32el \
 	| mipsisa32r2 | mipsisa32r2el \
 	| mipsisa64 | mipsisa64el \
 	| mipsisa64r2 | mipsisa64r2el \
 	| mipsisa64sb1 | mipsisa64sb1el \
 	| mipsisa64sr71k | mipsisa64sr71kel \
+	| mipsr5900 | mipsr5900el \
 	| mipstx39 | mipstx39el \
 	| mn10200 | mn10300 \
+	| moxie \
+	| mt \
 	| msp430 \
+	| nds32 | nds32le | nds32be \
+	| nios | nios2 | nios2eb | nios2el \
 	| ns16k | ns32k \
-	| openrisc | or32 \
+	| open8 \
+	| or1k | or32 \
 	| pdp10 | pdp11 | pj | pjl \
-	| powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \
+	| powerpc | powerpc64 | powerpc64le | powerpcle \
 	| pyramid \
-	| sh | sh[1234] | sh[23]e | sh[34]eb | shbe | shle | sh[1234]le | sh3ele \
+	| rl78 | rx \
+	| score \
+	| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
 	| sh64 | sh64le \
-	| sparc | sparc64 | sparc86x | sparclet | sparclite | sparcv9 | sparcv9b \
-	| strongarm \
-	| tahoe | thumb | tic4x | tic80 | tron \
-	| v850 | v850e \
+	| sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \
+	| sparcv8 | sparcv9 | sparcv9b | sparcv9v \
+	| spu \
+	| tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \
+	| ubicom32 \
+	| v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
 	| we32k \
-	| x86 | xscale | xstormy16 | xtensa \
-	| z8k)
+	| x86 | xc16x | xstormy16 | xtensa \
+	| z8k | z80)
 		basic_machine=$basic_machine-unknown
 		;;
-	m6811 | m68hc11 | m6812 | m68hc12)
-		# Motorola 68HC11/12.
+	c54x)
+		basic_machine=tic54x-unknown
+		;;
+	c55x)
+		basic_machine=tic55x-unknown
+		;;
+	c6x)
+		basic_machine=tic6x-unknown
+		;;
+	m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip)
 		basic_machine=$basic_machine-unknown
 		os=-none
 		;;
 	m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k)
 		;;
+	ms1)
+		basic_machine=mt-unknown
+		;;
+
+	strongarm | thumb | xscale)
+		basic_machine=arm-unknown
+		;;
+	xgate)
+		basic_machine=$basic_machine-unknown
+		os=-none
+		;;
+	xscaleeb)
+		basic_machine=armeb-unknown
+		;;
+
+	xscaleel)
+		basic_machine=armel-unknown
+		;;
 
 	# We use `pc' rather than `unknown'
 	# because (1) that's what they normally are, and
@@ -293,59 +364,83 @@ case $basic_machine in
 	# Recognize the basic CPU types with company name.
 	580-* \
 	| a29k-* \
+	| aarch64-* | aarch64_be-* \
 	| alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \
 	| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
-	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* \
+	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \
 	| arm-*  | armbe-* | armle-* | armeb-* | armv*-* \
-	| avr-* \
-	| bs2000-* \
-	| c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \
-	| clipper-* | cydra-* \
+	| avr-* | avr32-* \
+	| be32-* | be64-* \
+	| bfin-* | bs2000-* \
+	| c[123]* | c30-* | [cjt]90-* | c4x-* \
+	| c8051-* | clipper-* | craynv-* | cydra-* \
 	| d10v-* | d30v-* | dlx-* \
 	| elxsi-* \
-	| f30[01]-* | f700-* | fr30-* | frv-* | fx80-* \
+	| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \
 	| h8300-* | h8500-* \
 	| hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
+	| hexagon-* \
 	| i*86-* | i860-* | i960-* | ia64-* \
 	| ip2k-* | iq2000-* \
-	| m32r-* \
+	| k1om-* \
+	| le32-* | le64-* \
+	| lm32-* \
+	| m32c-* | m32r-* | m32rle-* \
 	| m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \
-	| m88110-* | m88k-* | mcore-* \
+	| m88110-* | m88k-* | maxq-* | mcore-* | metag-* \
+	| microblaze-* | microblazeel-* \
 	| mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \
 	| mips16-* \
 	| mips64-* | mips64el-* \
-	| mips64vr-* | mips64vrel-* \
+	| mips64octeon-* | mips64octeonel-* \
 	| mips64orion-* | mips64orionel-* \
+	| mips64r5900-* | mips64r5900el-* \
+	| mips64vr-* | mips64vrel-* \
 	| mips64vr4100-* | mips64vr4100el-* \
 	| mips64vr4300-* | mips64vr4300el-* \
 	| mips64vr5000-* | mips64vr5000el-* \
+	| mips64vr5900-* | mips64vr5900el-* \
 	| mipsisa32-* | mipsisa32el-* \
 	| mipsisa32r2-* | mipsisa32r2el-* \
 	| mipsisa64-* | mipsisa64el-* \
 	| mipsisa64r2-* | mipsisa64r2el-* \
 	| mipsisa64sb1-* | mipsisa64sb1el-* \
 	| mipsisa64sr71k-* | mipsisa64sr71kel-* \
+	| mipsr5900-* | mipsr5900el-* \
 	| mipstx39-* | mipstx39el-* \
+	| mmix-* \
+	| mt-* \
 	| msp430-* \
-	| none-* | np1-* | nv1-* | ns16k-* | ns32k-* \
+	| nds32-* | nds32le-* | nds32be-* \
+	| nios-* | nios2-* | nios2eb-* | nios2el-* \
+	| none-* | np1-* | ns16k-* | ns32k-* \
+	| open8-* \
 	| orion-* \
 	| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
-	| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \
+	| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \
 	| pyramid-* \
-	| romp-* | rs6000-* \
-	| sh-* | sh[1234]-* | sh[23]e-* | sh[34]eb-* | shbe-* \
+	| rl78-* | romp-* | rs6000-* | rx-* \
+	| sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \
 	| shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \
-	| sparc-* | sparc64-* | sparc86x-* | sparclet-* | sparclite-* \
-	| sparcv9-* | sparcv9b-* | strongarm-* | sv1-* | sx?-* \
-	| tahoe-* | thumb-* \
+	| sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \
+	| sparclite-* \
+	| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \
+	| tahoe-* \
 	| tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \
+	| tile*-* \
 	| tron-* \
-	| v850-* | v850e-* | vax-* \
+	| ubicom32-* \
+	| v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
+	| vax-* \
 	| we32k-* \
-	| x86-* | x86_64-* | xps100-* | xscale-* | xstormy16-* \
-	| xtensa-* \
+	| x86-* | x86_64-* | xc16x-* | xps100-* \
+	| xstormy16-* | xtensa*-* \
 	| ymp-* \
-	| z8k-*)
+	| z8k-* | z80-*)
+		;;
+	# Recognize the basic CPU types without company name, with glob match.
+	xtensa*)
+		basic_machine=$basic_machine-unknown
 		;;
 	# Recognize the various machine names and aliases which stand
 	# for a CPU type and a company and sometimes even an OS.
@@ -363,7 +458,7 @@ case $basic_machine in
 		basic_machine=a29k-amd
 		os=-udi
 		;;
-    	abacus)
+	abacus)
 		basic_machine=abacus-unknown
 		;;
 	adobe68k)
@@ -409,6 +504,10 @@ case $basic_machine in
 		basic_machine=m68k-apollo
 		os=-bsd
 		;;
+	aros)
+		basic_machine=i386-pc
+		os=-aros
+		;;
 	aux)
 		basic_machine=m68k-apple
 		os=-aux
@@ -417,10 +516,35 @@ case $basic_machine in
 		basic_machine=ns32k-sequent
 		os=-dynix
 		;;
+	blackfin)
+		basic_machine=bfin-unknown
+		os=-linux
+		;;
+	blackfin-*)
+		basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'`
+		os=-linux
+		;;
+	bluegene*)
+		basic_machine=powerpc-ibm
+		os=-cnk
+		;;
+	c54x-*)
+		basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	c55x-*)
+		basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	c6x-*)
+		basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
 	c90)
 		basic_machine=c90-cray
 		os=-unicos
 		;;
+	cegcc)
+		basic_machine=arm-unknown
+		os=-cegcc
+		;;
 	convex-c1)
 		basic_machine=c1-convex
 		os=-bsd
@@ -445,13 +569,20 @@ case $basic_machine in
 		basic_machine=j90-cray
 		os=-unicos
 		;;
-	cr16c)
-		basic_machine=cr16c-unknown
+	craynv)
+		basic_machine=craynv-cray
+		os=-unicosmp
+		;;
+	cr16 | cr16-*)
+		basic_machine=cr16-unknown
 		os=-elf
 		;;
 	crds | unos)
 		basic_machine=m68k-crds
 		;;
+	crisv32 | crisv32-* | etraxfs*)
+		basic_machine=crisv32-axis
+		;;
 	cris | cris-* | etrax*)
 		basic_machine=cris-axis
 		;;
@@ -481,6 +612,14 @@ case $basic_machine in
 		basic_machine=m88k-motorola
 		os=-sysv3
 		;;
+	dicos)
+		basic_machine=i686-pc
+		os=-dicos
+		;;
+	djgpp)
+		basic_machine=i586-pc
+		os=-msdosdjgpp
+		;;
 	dpx20 | dpx20-*)
 		basic_machine=rs6000-bull
 		os=-bosx
@@ -592,7 +731,6 @@ case $basic_machine in
 	i370-ibm* | ibm*)
 		basic_machine=i370-ibm
 		;;
-# I'm not sure what "Sysv32" means.  Should this be sysv3.2?
 	i*86v32)
 		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
 		os=-sysv32
@@ -631,6 +769,14 @@ case $basic_machine in
 		basic_machine=m68k-isi
 		os=-sysv
 		;;
+	m68knommu)
+		basic_machine=m68k-unknown
+		os=-linux
+		;;
+	m68knommu-*)
+		basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'`
+		os=-linux
+		;;
 	m88k-omron*)
 		basic_machine=m88k-omron
 		;;
@@ -642,10 +788,21 @@ case $basic_machine in
 		basic_machine=ns32k-utek
 		os=-sysv
 		;;
+	microblaze*)
+		basic_machine=microblaze-xilinx
+		;;
+	mingw64)
+		basic_machine=x86_64-pc
+		os=-mingw64
+		;;
 	mingw32)
-		basic_machine=i386-pc
+		basic_machine=i686-pc
 		os=-mingw32
 		;;
+	mingw32ce)
+		basic_machine=arm-unknown
+		os=-mingw32ce
+		;;
 	miniframe)
 		basic_machine=m68000-convergent
 		;;
@@ -659,10 +816,6 @@ case $basic_machine in
 	mips3*)
 		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
 		;;
-	mmix*)
-		basic_machine=mmix-knuth
-		os=-mmixware
-		;;
 	monitor)
 		basic_machine=m68k-rom68k
 		os=-coff
@@ -675,10 +828,21 @@ case $basic_machine in
 		basic_machine=i386-pc
 		os=-msdos
 		;;
+	ms1-*)
+		basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'`
+		;;
+	msys)
+		basic_machine=i686-pc
+		os=-msys
+		;;
 	mvs)
 		basic_machine=i370-ibm
 		os=-mvs
 		;;
+	nacl)
+		basic_machine=le32-unknown
+		os=-nacl
+		;;
 	ncr3000)
 		basic_machine=i486-ncr
 		os=-sysv4
@@ -743,9 +907,11 @@ case $basic_machine in
 	np1)
 		basic_machine=np1-gould
 		;;
-	nv1)
-		basic_machine=nv1-cray
-		os=-unicosmp
+	neo-tandem)
+		basic_machine=neo-tandem
+		;;
+	nse-tandem)
+		basic_machine=nse-tandem
 		;;
 	nsr-tandem)
 		basic_machine=nsr-tandem
@@ -754,9 +920,8 @@ case $basic_machine in
 		basic_machine=hppa1.1-oki
 		os=-proelf
 		;;
-	or32 | or32-*)
+	openrisc | openrisc-*)
 		basic_machine=or32-unknown
-		os=-coff
 		;;
 	os400)
 		basic_machine=powerpc-ibm
@@ -778,6 +943,14 @@ case $basic_machine in
 		basic_machine=i860-intel
 		os=-osf
 		;;
+	parisc)
+		basic_machine=hppa-unknown
+		os=-linux
+		;;
+	parisc-*)
+		basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'`
+		os=-linux
+		;;
 	pbd)
 		basic_machine=sparc-tti
 		;;
@@ -787,6 +960,12 @@ case $basic_machine in
 	pc532 | pc532-*)
 		basic_machine=ns32k-pc532
 		;;
+	pc98)
+		basic_machine=i386-pc
+		;;
+	pc98-*)
+		basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
 	pentium | p5 | k5 | k6 | nexgen | viac3)
 		basic_machine=i586-pc
 		;;
@@ -816,9 +995,10 @@ case $basic_machine in
 		;;
 	power)	basic_machine=power-ibm
 		;;
-	ppc)	basic_machine=powerpc-unknown
+	ppc | ppcbe)	basic_machine=powerpc-unknown
 		;;
-	ppc-*)	basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
+	ppc-* | ppcbe-*)
+		basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
 		;;
 	ppcle | powerpclittle | ppc-le | powerpc-little)
 		basic_machine=powerpcle-unknown
@@ -843,6 +1023,14 @@ case $basic_machine in
 		basic_machine=i586-unknown
 		os=-pw32
 		;;
+	rdos | rdos64)
+		basic_machine=x86_64-pc
+		os=-rdos
+		;;
+	rdos32)
+		basic_machine=i386-pc
+		os=-rdos
+		;;
 	rom68k)
 		basic_machine=m68k-rom68k
 		os=-coff
@@ -869,6 +1057,10 @@ case $basic_machine in
 	sb1el)
 		basic_machine=mipsisa64sb1el-unknown
 		;;
+	sde)
+		basic_machine=mipsisa32-sde
+		os=-elf
+		;;
 	sei)
 		basic_machine=mips-sei
 		os=-seiux
@@ -880,6 +1072,9 @@ case $basic_machine in
 		basic_machine=sh-hitachi
 		os=-hms
 		;;
+	sh5el)
+		basic_machine=sh5le-unknown
+		;;
 	sh64)
 		basic_machine=sh64-unknown
 		;;
@@ -901,6 +1096,9 @@ case $basic_machine in
 		basic_machine=i860-stratus
 		os=-sysv4
 		;;
+	strongarm-* | thumb-*)
+		basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
 	sun2)
 		basic_machine=m68000-sun
 		;;
@@ -957,17 +1155,9 @@ case $basic_machine in
 		basic_machine=t90-cray
 		os=-unicos
 		;;
-	tic54x | c54x*)
-		basic_machine=tic54x-unknown
-		os=-coff
-		;;
-	tic55x | c55x*)
-		basic_machine=tic55x-unknown
-		os=-coff
-		;;
-	tic6x | c6x*)
-		basic_machine=tic6x-unknown
-		os=-coff
+	tile*)
+		basic_machine=$basic_machine-unknown
+		os=-linux-gnu
 		;;
 	tx39)
 		basic_machine=mipstx39-unknown
@@ -1029,9 +1219,16 @@ case $basic_machine in
 		basic_machine=hppa1.1-winbond
 		os=-proelf
 		;;
+	xbox)
+		basic_machine=i686-pc
+		os=-mingw32
+		;;
 	xps | xps100)
 		basic_machine=xps100-honeywell
 		;;
+	xscale-* | xscalee[bl]-*)
+		basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'`
+		;;
 	ymp)
 		basic_machine=ymp-cray
 		os=-unicos
@@ -1040,6 +1237,10 @@ case $basic_machine in
 		basic_machine=z8k-unknown
 		os=-sim
 		;;
+	z80-*-coff)
+		basic_machine=z80-unknown
+		os=-sim
+		;;
 	none)
 		basic_machine=none-none
 		os=-none
@@ -1059,6 +1260,9 @@ case $basic_machine in
 	romp)
 		basic_machine=romp-ibm
 		;;
+	mmix)
+		basic_machine=mmix-knuth
+		;;
 	rs6000)
 		basic_machine=rs6000-ibm
 		;;
@@ -1075,13 +1279,10 @@ case $basic_machine in
 	we32k)
 		basic_machine=we32k-att
 		;;
-	sh3 | sh4 | sh[34]eb | sh[1234]le | sh[23]ele)
+	sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele)
 		basic_machine=sh-unknown
 		;;
-	sh64)
-		basic_machine=sh64-unknown
-		;;
-	sparc | sparcv9 | sparcv9b)
+	sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v)
 		basic_machine=sparc-sun
 		;;
 	cydra)
@@ -1125,9 +1326,12 @@ esac
 if [ x"$os" != x"" ]
 then
 case $os in
-        # First match some system type aliases
-        # that might get confused with valid system types.
+	# First match some system type aliases
+	# that might get confused with valid system types.
 	# -solaris* is a basic system type, with this one exception.
+	-auroraux)
+		os=-auroraux
+		;;
 	-solaris1 | -solaris1.*)
 		os=`echo $os | sed -e 's|solaris1|sunos4|'`
 		;;
@@ -1148,26 +1352,31 @@ case $os in
 	# Each alternative MUST END IN A *, to match a version number.
 	# -sysv* is not here because it comes later, after sysvr4.
 	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
-	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\
-	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \
+	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
+	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
+	      | -sym* | -kopensolaris* | -plan9* \
 	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
-	      | -aos* \
+	      | -aos* | -aros* \
 	      | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
 	      | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
-	      | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* | -openbsd* \
+	      | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \
+	      | -bitrig* | -openbsd* | -solidbsd* \
 	      | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \
 	      | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
 	      | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
 	      | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
-	      | -chorusos* | -chorusrdb* \
-	      | -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
-	      | -mingw32* | -linux-gnu* | -linux-uclibc* | -uxpv* | -beos* | -mpeix* | -udk* \
+	      | -chorusos* | -chorusrdb* | -cegcc* \
+	      | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
+	      | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \
+	      | -linux-newlib* | -linux-musl* | -linux-uclibc* \
+	      | -uxpv* | -beos* | -mpeix* | -udk* \
 	      | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \
 	      | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
 	      | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \
 	      | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \
 	      | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
-	      | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly*)
+	      | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
+	      | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*)
 	# Remember, each alternative MUST END IN *, to match a version number.
 		;;
 	-qnx*)
@@ -1185,7 +1394,7 @@ case $os in
 		os=`echo $os | sed -e 's|nto|nto-qnx|'`
 		;;
 	-sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \
-	      | -windows* | -osx | -abug | -netware* | -os9* | -beos* \
+	      | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \
 	      | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*)
 		;;
 	-mac*)
@@ -1206,7 +1415,7 @@ case $os in
 	-opened*)
 		os=-openedition
 		;;
-        -os400*)
+	-os400*)
 		os=-os400
 		;;
 	-wince*)
@@ -1255,7 +1464,7 @@ case $os in
 	-sinix*)
 		os=-sysv4
 		;;
-        -tpf*)
+	-tpf*)
 		os=-tpf
 		;;
 	-triton*)
@@ -1291,8 +1500,13 @@ case $os in
 	-aros*)
 		os=-aros
 		;;
-	-kaos*)
-		os=-kaos
+	-zvmoe)
+		os=-zvmoe
+		;;
+	-dicos*)
+		os=-dicos
+		;;
+	-nacl*)
 		;;
 	-none)
 		;;
@@ -1316,6 +1530,12 @@ else
 # system, and we'll never get to this point.
 
 case $basic_machine in
+	score-*)
+		os=-elf
+		;;
+	spu-*)
+		os=-elf
+		;;
 	*-acorn)
 		os=-riscix1.2
 		;;
@@ -1325,9 +1545,24 @@ case $basic_machine in
 	arm*-semi)
 		os=-aout
 		;;
-    c4x-* | tic4x-*)
-        os=-coff
-        ;;
+	c4x-* | tic4x-*)
+		os=-coff
+		;;
+	c8051-*)
+		os=-elf
+		;;
+	hexagon-*)
+		os=-elf
+		;;
+	tic54x-*)
+		os=-coff
+		;;
+	tic55x-*)
+		os=-coff
+		;;
+	tic6x-*)
+		os=-coff
+		;;
 	# This must come before the *-dec entry.
 	pdp10-*)
 		os=-tops20
@@ -1346,19 +1581,22 @@ case $basic_machine in
 		;;
 	m68000-sun)
 		os=-sunos3
-		# This also exists in the configure program, but was not the
-		# default.
-		# os=-sunos4
 		;;
 	m68*-cisco)
 		os=-aout
 		;;
+	mep-*)
+		os=-elf
+		;;
 	mips*-cisco)
 		os=-elf
 		;;
 	mips*-*)
 		os=-elf
 		;;
+	or1k-*)
+		os=-elf
+		;;
 	or32-*)
 		os=-coff
 		;;
@@ -1371,9 +1609,15 @@ case $basic_machine in
 	*-be)
 		os=-beos
 		;;
+	*-haiku)
+		os=-haiku
+		;;
 	*-ibm)
 		os=-aix
 		;;
+	*-knuth)
+		os=-mmixware
+		;;
 	*-wec)
 		os=-proelf
 		;;
@@ -1476,7 +1720,7 @@ case $basic_machine in
 			-sunos*)
 				vendor=sun
 				;;
-			-aix*)
+			-cnk*|-aix*)
 				vendor=ibm
 				;;
 			-beos*)
@@ -1539,7 +1783,7 @@ case $basic_machine in
 esac
 
 echo $basic_machine$os
-exit 0
+exit
 
 # Local variables:
 # eval: (add-hook 'write-file-hooks 'time-stamp)

coverage.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+objdir=$1
+suffix=$2
+shift 2
+objs=$@
+
+gcov -b -p -f -o "${objdir}" ${objs}
+
+# Move gcov outputs so that subsequent gcov invocations won't clobber results
+# for the same sources with different compilation flags.
+for f in `find . -maxdepth 1 -type f -name '*.gcov'` ; do
+  mv "${f}" "${f}.${suffix}"
+done

include/jemalloc/internal/atomic.h

@@ -11,22 +11,8 @@
 
 #define	atomic_read_uint64(p)	atomic_add_uint64(p, 0)
 #define	atomic_read_uint32(p)	atomic_add_uint32(p, 0)
-
-#if (LG_SIZEOF_PTR == 3)
-#  define atomic_read_z(p)						\
-    (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)0)
-#  define atomic_add_z(p, x)						\
-    (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x)
-#  define atomic_sub_z(p, x)						\
-    (size_t)atomic_sub_uint64((uint64_t *)p, (uint64_t)x)
-#elif (LG_SIZEOF_PTR == 2)
-#  define atomic_read_z(p)						\
-    (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)0)
-#  define atomic_add_z(p, x)						\
-    (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x)
-#  define atomic_sub_z(p, x)						\
-    (size_t)atomic_sub_uint32((uint32_t *)p, (uint32_t)x)
-#endif
+#define	atomic_read_z(p)	atomic_add_z(p, 0)
+#define	atomic_read_u(p)	atomic_add_u(p, 0)
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
@@ -37,12 +23,17 @@ uint64_t	atomic_add_uint64(uint64_t *p, uint64_t x);
 uint64_t	atomic_sub_uint64(uint64_t *p, uint64_t x);
 uint32_t	atomic_add_uint32(uint32_t *p, uint32_t x);
 uint32_t	atomic_sub_uint32(uint32_t *p, uint32_t x);
+size_t	atomic_add_z(size_t *p, size_t x);
+size_t	atomic_sub_z(size_t *p, size_t x);
+unsigned	atomic_add_u(unsigned *p, unsigned x);
+unsigned	atomic_sub_u(unsigned *p, unsigned x);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))
 /******************************************************************************/
 /* 64-bit operations. */
-#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
+#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
+#  ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
 JEMALLOC_INLINE uint64_t
 atomic_add_uint64(uint64_t *p, uint64_t x)
 {
@@ -56,6 +47,20 @@ atomic_sub_uint64(uint64_t *p, uint64_t x)
 
 	return (__sync_sub_and_fetch(p, x));
 }
+#elif (defined(_MSC_VER))
+JEMALLOC_INLINE uint64_t
+atomic_add_uint64(uint64_t *p, uint64_t x)
+{
+
+	return (InterlockedExchangeAdd64(p, x));
+}
+
+JEMALLOC_INLINE uint64_t
+atomic_sub_uint64(uint64_t *p, uint64_t x)
+{
+
+	return (InterlockedExchangeAdd64(p, -((int64_t)x)));
+}
 #elif (defined(JEMALLOC_OSATOMIC))
 JEMALLOC_INLINE uint64_t
 atomic_add_uint64(uint64_t *p, uint64_t x)
@@ -70,7 +75,7 @@ atomic_sub_uint64(uint64_t *p, uint64_t x)
 
 	return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
 }
-#elif (defined(__amd64_) || defined(__x86_64__))
+#  elif (defined(__amd64__) || defined(__x86_64__))
 JEMALLOC_INLINE uint64_t
 atomic_add_uint64(uint64_t *p, uint64_t x)
 {
@@ -97,8 +102,43 @@ atomic_sub_uint64(uint64_t *p, uint64_t x)
 
 	return (x);
 }
-#else
-#  if (LG_SIZEOF_PTR == 3)
+#  elif (defined(JEMALLOC_ATOMIC9))
+JEMALLOC_INLINE uint64_t
+atomic_add_uint64(uint64_t *p, uint64_t x)
+{
+
+	/*
+	 * atomic_fetchadd_64() doesn't exist, but we only ever use this
+	 * function on LP64 systems, so atomic_fetchadd_long() will do.
+	 */
+	assert(sizeof(uint64_t) == sizeof(unsigned long));
+
+	return (atomic_fetchadd_long(p, (unsigned long)x) + x);
+}
+
+JEMALLOC_INLINE uint64_t
+atomic_sub_uint64(uint64_t *p, uint64_t x)
+{
+
+	assert(sizeof(uint64_t) == sizeof(unsigned long));
+
+	return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x);
+}
+#  elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
+JEMALLOC_INLINE uint64_t
+atomic_add_uint64(uint64_t *p, uint64_t x)
+{
+
+	return (__sync_add_and_fetch(p, x));
+}
+
+JEMALLOC_INLINE uint64_t
+atomic_sub_uint64(uint64_t *p, uint64_t x)
+{
+
+	return (__sync_sub_and_fetch(p, x));
+}
+#  else
 #    error "Missing implementation for 64-bit atomic operations"
 #  endif
 #endif
@@ -119,6 +159,20 @@ atomic_sub_uint32(uint32_t *p, uint32_t x)
 
 	return (__sync_sub_and_fetch(p, x));
 }
+#elif (defined(_MSC_VER))
+JEMALLOC_INLINE uint32_t
+atomic_add_uint32(uint32_t *p, uint32_t x)
+{
+
+	return (InterlockedExchangeAdd(p, x));
+}
+
+JEMALLOC_INLINE uint32_t
+atomic_sub_uint32(uint32_t *p, uint32_t x)
+{
+
+	return (InterlockedExchangeAdd(p, -((int32_t)x)));
+}
 #elif (defined(JEMALLOC_OSATOMIC))
 JEMALLOC_INLINE uint32_t
 atomic_add_uint32(uint32_t *p, uint32_t x)
@@ -133,7 +187,7 @@ atomic_sub_uint32(uint32_t *p, uint32_t x)
 
 	return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p));
 }
-#elif (defined(__i386__) || defined(__amd64_) || defined(__x86_64__))
+#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
 JEMALLOC_INLINE uint32_t
 atomic_add_uint32(uint32_t *p, uint32_t x)
 {
@@ -160,9 +214,90 @@ atomic_sub_uint32(uint32_t *p, uint32_t x)
 
 	return (x);
 }
+#elif (defined(JEMALLOC_ATOMIC9))
+JEMALLOC_INLINE uint32_t
+atomic_add_uint32(uint32_t *p, uint32_t x)
+{
+
+	return (atomic_fetchadd_32(p, x) + x);
+}
+
+JEMALLOC_INLINE uint32_t
+atomic_sub_uint32(uint32_t *p, uint32_t x)
+{
+
+	return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);
+}
+#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
+JEMALLOC_INLINE uint32_t
+atomic_add_uint32(uint32_t *p, uint32_t x)
+{
+
+	return (__sync_add_and_fetch(p, x));
+}
+
+JEMALLOC_INLINE uint32_t
+atomic_sub_uint32(uint32_t *p, uint32_t x)
+{
+
+	return (__sync_sub_and_fetch(p, x));
+}
 #else
 #  error "Missing implementation for 32-bit atomic operations"
 #endif
+
+/******************************************************************************/
+/* size_t operations. */
+JEMALLOC_INLINE size_t
+atomic_add_z(size_t *p, size_t x)
+{
+
+#if (LG_SIZEOF_PTR == 3)
+	return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
+#elif (LG_SIZEOF_PTR == 2)
+	return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
+#endif
+}
+
+JEMALLOC_INLINE size_t
+atomic_sub_z(size_t *p, size_t x)
+{
+
+#if (LG_SIZEOF_PTR == 3)
+	return ((size_t)atomic_add_uint64((uint64_t *)p,
+	    (uint64_t)-((int64_t)x)));
+#elif (LG_SIZEOF_PTR == 2)
+	return ((size_t)atomic_add_uint32((uint32_t *)p,
+	    (uint32_t)-((int32_t)x)));
+#endif
+}
+
+/******************************************************************************/
+/* unsigned operations. */
+JEMALLOC_INLINE unsigned
+atomic_add_u(unsigned *p, unsigned x)
+{
+
+#if (LG_SIZEOF_INT == 3)
+	return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
+#elif (LG_SIZEOF_INT == 2)
+	return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
+#endif
+}
+
+JEMALLOC_INLINE unsigned
+atomic_sub_u(unsigned *p, unsigned x)
+{
+
+#if (LG_SIZEOF_INT == 3)
+	return ((unsigned)atomic_add_uint64((uint64_t *)p,
+	    (uint64_t)-((int64_t)x)));
+#elif (LG_SIZEOF_INT == 2)
+	return ((unsigned)atomic_add_uint32((uint32_t *)p,
+	    (uint32_t)-((int32_t)x)));
+#endif
+}
+/******************************************************************************/
 #endif
 
 #endif /* JEMALLOC_H_INLINES */

include/jemalloc/internal/base.h

@@ -9,12 +9,14 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-extern malloc_mutex_t	base_mtx;
-
 void	*base_alloc(size_t size);
+void	*base_calloc(size_t number, size_t size);
 extent_node_t *base_node_alloc(void);
 void	base_node_dealloc(extent_node_t *node);
 bool	base_boot(void);
+void	base_prefork(void);
+void	base_postfork_parent(void);
+void	base_postfork_child(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/

include/jemalloc/internal/chunk.h

@@ -28,20 +28,14 @@
 #ifdef JEMALLOC_H_EXTERNS
 
 extern size_t		opt_lg_chunk;
-#ifdef JEMALLOC_SWAP
-extern bool		opt_overcommit;
-#endif
+extern const char	*opt_dss;
 
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
 /* Protects stats_chunks; currently not used for any other purpose. */
 extern malloc_mutex_t	chunks_mtx;
 /* Chunk statistics. */
 extern chunk_stats_t	stats_chunks;
-#endif
 
-#ifdef JEMALLOC_IVSALLOC
 extern rtree_t		*chunks_rtree;
-#endif
 
 extern size_t		chunksize;
 extern size_t		chunksize_mask; /* (chunksize - 1). */
@@ -49,9 +43,14 @@ extern size_t		chunk_npages;
 extern size_t		map_bias; /* Number of arena chunk header pages. */
 extern size_t		arena_maxclass; /* Max size class for arenas. */
 
-void	*chunk_alloc(size_t size, bool base, bool *zero);
-void	chunk_dealloc(void *chunk, size_t size);
+void	*chunk_alloc(size_t size, size_t alignment, bool base, bool *zero,
+    dss_prec_t dss_prec);
+void	chunk_unmap(void *chunk, size_t size);
+void	chunk_dealloc(void *chunk, size_t size, bool unmap);
 bool	chunk_boot(void);
+void	chunk_prefork(void);
+void	chunk_postfork_parent(void);
+void	chunk_postfork_child(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
@@ -60,6 +59,5 @@ bool	chunk_boot(void);
 #endif /* JEMALLOC_H_INLINES */
 /******************************************************************************/
 
-#include "jemalloc/internal/chunk_swap.h"
 #include "jemalloc/internal/chunk_dss.h"
 #include "jemalloc/internal/chunk_mmap.h"

include/jemalloc/internal/chunk_dss.h

@@ -1,25 +1,34 @@
-#ifdef JEMALLOC_DSS
 /******************************************************************************/
 #ifdef JEMALLOC_H_TYPES
 
+typedef enum {
+	dss_prec_disabled  = 0,
+	dss_prec_primary   = 1,
+	dss_prec_secondary = 2,
+
+	dss_prec_limit     = 3
+} dss_prec_t;
+#define	DSS_PREC_DEFAULT	dss_prec_secondary
+#define	DSS_DEFAULT		"secondary"
+
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
 
+extern const char *dss_prec_names[];
+
 #endif /* JEMALLOC_H_STRUCTS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-/*
- * Protects sbrk() calls.  This avoids malloc races among threads, though it
- * does not protect against races with threads that call sbrk() directly.
- */
-extern malloc_mutex_t	dss_mtx;
-
-void	*chunk_alloc_dss(size_t size, bool *zero);
+dss_prec_t	chunk_dss_prec_get(void);
+bool	chunk_dss_prec_set(dss_prec_t dss_prec);
+void	*chunk_alloc_dss(size_t size, size_t alignment, bool *zero);
 bool	chunk_in_dss(void *chunk);
-bool	chunk_dealloc_dss(void *chunk, size_t size);
 bool	chunk_dss_boot(void);
+void	chunk_dss_prefork(void);
+void	chunk_dss_postfork_parent(void);
+void	chunk_dss_postfork_child(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
@@ -27,4 +36,3 @@ bool	chunk_dss_boot(void);
 
 #endif /* JEMALLOC_H_INLINES */
 /******************************************************************************/
-#endif /* JEMALLOC_DSS */

include/jemalloc/internal/chunk_mmap.h

@@ -9,11 +9,10 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-void	*chunk_alloc_mmap(size_t size);
-void	*chunk_alloc_mmap_noreserve(size_t size);
-void	chunk_dealloc_mmap(void *chunk, size_t size);
+bool	pages_purge(void *addr, size_t length);
 
-bool	chunk_mmap_boot(void);
+void	*chunk_alloc_mmap(size_t size, size_t alignment, bool *zero);
+bool	chunk_dealloc_mmap(void *chunk, size_t size);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/

include/jemalloc/internal/chunk_swap.h

@@ -1,34 +0,0 @@
-#ifdef JEMALLOC_SWAP
-/******************************************************************************/
-#ifdef JEMALLOC_H_TYPES
-
-#endif /* JEMALLOC_H_TYPES */
-/******************************************************************************/
-#ifdef JEMALLOC_H_STRUCTS
-
-#endif /* JEMALLOC_H_STRUCTS */
-/******************************************************************************/
-#ifdef JEMALLOC_H_EXTERNS
-
-extern malloc_mutex_t	swap_mtx;
-extern bool		swap_enabled;
-extern bool		swap_prezeroed;
-extern size_t		swap_nfds;
-extern int		*swap_fds;
-#ifdef JEMALLOC_STATS
-extern size_t		swap_avail;
-#endif
-
-void	*chunk_alloc_swap(size_t size, bool *zero);
-bool	chunk_in_swap(void *chunk);
-bool	chunk_dealloc_swap(void *chunk, size_t size);
-bool	chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed);
-bool	chunk_swap_boot(void);
-
-#endif /* JEMALLOC_H_EXTERNS */
-/******************************************************************************/
-#ifdef JEMALLOC_H_INLINES
-
-#endif /* JEMALLOC_H_INLINES */
-/******************************************************************************/
-#endif /* JEMALLOC_SWAP */

include/jemalloc/internal/ckh.h

@@ -5,7 +5,7 @@ typedef struct ckh_s ckh_t;
 typedef struct ckhc_s ckhc_t;
 
 /* Typedefs to allow easy function pointer passing. */
-typedef void ckh_hash_t (const void *, unsigned, size_t *, size_t *);
+typedef void ckh_hash_t (const void *, size_t[2]);
 typedef bool ckh_keycomp_t (const void *, const void *);
 
 /* Maintain counters used to get an idea of performance. */
@@ -17,7 +17,7 @@ typedef bool ckh_keycomp_t (const void *, const void *);
  * There are 2^LG_CKH_BUCKET_CELLS cells in each hash table bucket.  Try to fit
  * one bucket per L1 cache line.
  */
-#define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1)
+#define	LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1)
 
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
@@ -30,11 +30,6 @@ struct ckhc_s {
 };
 
 struct ckh_s {
-#ifdef JEMALLOC_DEBUG
-#define	CKH_MAGIC	0x3af2489d
-	uint32_t	magic;
-#endif
-
 #ifdef CKH_COUNT
 	/* Counters used to get an idea of performance. */
 	uint64_t	ngrows;
@@ -47,7 +42,7 @@ struct ckh_s {
 	/* Used for pseudo-random number generation. */
 #define	CKH_A		1103515241
 #define	CKH_C		12347
-	uint32_t	prn_state;
+	uint32_t	prng_state;
 
 	/* Total number of items. */
 	size_t		count;
@@ -80,11 +75,9 @@ bool	ckh_insert(ckh_t *ckh, const void *key, const void *data);
 bool	ckh_remove(ckh_t *ckh, const void *searchkey, void **key,
     void **data);
 bool	ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data);
-void	ckh_string_hash(const void *key, unsigned minbits, size_t *hash1,
-    size_t *hash2);
+void	ckh_string_hash(const void *key, size_t r_hash[2]);
 bool	ckh_string_keycomp(const void *k1, const void *k2);
-void	ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1,
-    size_t *hash2);
+void	ckh_pointer_hash(const void *key, size_t r_hash[2]);
 bool	ckh_pointer_keycomp(const void *k1, const void *k2);
 
 #endif /* JEMALLOC_H_EXTERNS */

include/jemalloc/internal/ctl.h

@@ -2,6 +2,8 @@
 #ifdef JEMALLOC_H_TYPES
 
 typedef struct ctl_node_s ctl_node_t;
+typedef struct ctl_named_node_s ctl_named_node_t;
+typedef struct ctl_indexed_node_s ctl_indexed_node_t;
 typedef struct ctl_arena_stats_s ctl_arena_stats_t;
 typedef struct ctl_stats_s ctl_stats_t;
 
@@ -11,28 +13,29 @@ typedef struct ctl_stats_s ctl_stats_t;
 
 struct ctl_node_s {
 	bool			named;
-	union {
-		struct {
-			const char	*name;
-			/* If (nchildren == 0), this is a terminal node. */
-			unsigned	nchildren;
-			const	ctl_node_t *children;
-		} named;
-		struct {
-			const ctl_node_t *(*index)(const size_t *, size_t,
-			    size_t);
-		} indexed;
-	} u;
-	int	(*ctl)(const size_t *, size_t, void *, size_t *, void *,
-	    size_t);
+};
+
+struct ctl_named_node_s {
+	struct ctl_node_s	node;
+	const char		*name;
+	/* If (nchildren == 0), this is a terminal node. */
+	unsigned		nchildren;
+	const			ctl_node_t *children;
+	int			(*ctl)(const size_t *, size_t, void *, size_t *,
+	    void *, size_t);
+};
+
+struct ctl_indexed_node_s {
+	struct ctl_node_s	node;
+	const ctl_named_node_t	*(*index)(const size_t *, size_t, size_t);
 };
 
 struct ctl_arena_stats_s {
 	bool			initialized;
 	unsigned		nthreads;
+	const char		*dss;
 	size_t			pactive;
 	size_t			pdirty;
-#ifdef JEMALLOC_STATS
 	arena_stats_t		astats;
 
 	/* Aggregate stats for small size classes, based on bin stats. */
@@ -41,13 +44,11 @@ struct ctl_arena_stats_s {
 	uint64_t		ndalloc_small;
 	uint64_t		nrequests_small;
 
-	malloc_bin_stats_t	*bstats;	/* nbins elements. */
+	malloc_bin_stats_t	bstats[NBINS];
 	malloc_large_stats_t	*lstats;	/* nlclasses elements. */
-#endif
 };
 
 struct ctl_stats_s {
-#ifdef JEMALLOC_STATS
 	size_t			allocated;
 	size_t			active;
 	size_t			mapped;
@@ -61,11 +62,8 @@ struct ctl_stats_s {
 		uint64_t	nmalloc;	/* huge_nmalloc */
 		uint64_t	ndalloc;	/* huge_ndalloc */
 	} huge;
-#endif
+	unsigned		narenas;
 	ctl_arena_stats_t	*arenas;	/* (narenas + 1) elements. */
-#ifdef JEMALLOC_SWAP
-	size_t			swap_avail;
-#endif
 };
 
 #endif /* JEMALLOC_H_STRUCTS */
@@ -79,29 +77,30 @@ int	ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp);
 int	ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
     void *newp, size_t newlen);
 bool	ctl_boot(void);
+void	ctl_prefork(void);
+void	ctl_postfork_parent(void);
+void	ctl_postfork_child(void);
 
 #define	xmallctl(name, oldp, oldlenp, newp, newlen) do {		\
-	if (JEMALLOC_P(mallctl)(name, oldp, oldlenp, newp, newlen)	\
+	if (je_mallctl(name, oldp, oldlenp, newp, newlen)		\
 	    != 0) {							\
-		malloc_write("<jemalloc>: Failure in xmallctl(\"");	\
-		malloc_write(name);					\
-		malloc_write("\", ...)\n");				\
+		malloc_printf(						\
+		    "<jemalloc>: Failure in xmallctl(\"%s\", ...)\n",	\
+		    name);						\
 		abort();						\
 	}								\
 } while (0)
 
 #define	xmallctlnametomib(name, mibp, miblenp) do {			\
-	if (JEMALLOC_P(mallctlnametomib)(name, mibp, miblenp) != 0) {	\
-		malloc_write(						\
-		    "<jemalloc>: Failure in xmallctlnametomib(\"");	\
-		malloc_write(name);					\
-		malloc_write("\", ...)\n");				\
+	if (je_mallctlnametomib(name, mibp, miblenp) != 0) {		\
+		malloc_printf("<jemalloc>: Failure in "			\
+		    "xmallctlnametomib(\"%s\", ...)\n", name);		\
 		abort();						\
 	}								\
 } while (0)
 
 #define	xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) do {	\
-	if (JEMALLOC_P(mallctlbymib)(mib, miblen, oldp, oldlenp, newp,	\
+	if (je_mallctlbymib(mib, miblen, oldp, oldlenp, newp,		\
 	    newlen) != 0) {						\
 		malloc_write(						\
 		    "<jemalloc>: Failure in xmallctlbymib()\n");	\

include/jemalloc/internal/extent.h

@@ -9,24 +9,23 @@ typedef struct extent_node_s extent_node_t;
 
 /* Tree of extents. */
 struct extent_node_s {
-#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS))
 	/* Linkage for the size/address-ordered tree. */
 	rb_node(extent_node_t)	link_szad;
-#endif
 
 	/* Linkage for the address-ordered tree. */
 	rb_node(extent_node_t)	link_ad;
 
-#ifdef JEMALLOC_PROF
 	/* Profile counters, used for huge objects. */
 	prof_ctx_t		*prof_ctx;
-#endif
 
 	/* Pointer to the extent that this tree node is responsible for. */
 	void			*addr;
 
 	/* Total region size. */
 	size_t			size;
+
+	/* True if zero-filled; used by chunk recycling code. */
+	bool			zeroed;
 };
 typedef rb_tree(extent_node_t) extent_tree_t;
 
@@ -34,9 +33,7 @@ typedef rb_tree(extent_node_t) extent_tree_t;
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS))
 rb_proto(, extent_tree_szad_, extent_tree_t, extent_node_t)
-#endif
 
 rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t)
 

include/jemalloc/internal/hash.h

@@ -1,3 +1,8 @@
+/*
+ * The following hash function is based on MurmurHash3, placed into the public
+ * domain by Austin Appleby.  See http://code.google.com/p/smhasher/ for
+ * details.
+ */
 /******************************************************************************/
 #ifdef JEMALLOC_H_TYPES
 
@@ -14,56 +19,316 @@
 #ifdef JEMALLOC_H_INLINES
 
 #ifndef JEMALLOC_ENABLE_INLINE
-uint64_t	hash(const void *key, size_t len, uint64_t seed);
+uint32_t	hash_x86_32(const void *key, int len, uint32_t seed);
+void	hash_x86_128(const void *key, const int len, uint32_t seed,
+    uint64_t r_out[2]);
+void	hash_x64_128(const void *key, const int len, const uint32_t seed,
+    uint64_t r_out[2]);
+void	hash(const void *key, size_t len, const uint32_t seed,
+    size_t r_hash[2]);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_HASH_C_))
-/*
- * The following hash function is based on MurmurHash64A(), placed into the
- * public domain by Austin Appleby.  See http://murmurhash.googlepages.com/ for
- * details.
- */
-JEMALLOC_INLINE uint64_t
-hash(const void *key, size_t len, uint64_t seed)
+/******************************************************************************/
+/* Internal implementation. */
+JEMALLOC_INLINE uint32_t
+hash_rotl_32(uint32_t x, int8_t r)
 {
-	const uint64_t m = 0xc6a4a7935bd1e995LLU;
-	const int r = 47;
-	uint64_t h = seed ^ (len * m);
-	const uint64_t *data = (const uint64_t *)key;
-	const uint64_t *end = data + (len/8);
-	const unsigned char *data2;
 
-	assert(((uintptr_t)key & 0x7) == 0);
+	return (x << r) | (x >> (32 - r));
+}
 
-	while(data != end) {
-		uint64_t k = *data++;
+JEMALLOC_INLINE uint64_t
+hash_rotl_64(uint64_t x, int8_t r)
+{
+	return (x << r) | (x >> (64 - r));
+}
 
-		k *= m;
-		k ^= k >> r;
-		k *= m;
+JEMALLOC_INLINE uint32_t
+hash_get_block_32(const uint32_t *p, int i)
+{
 
-		h ^= k;
-		h *= m;
-	}
+	return (p[i]);
+}
 
-	data2 = (const unsigned char *)data;
-	switch(len & 7) {
-		case 7: h ^= ((uint64_t)(data2[6])) << 48;
-		case 6: h ^= ((uint64_t)(data2[5])) << 40;
-		case 5: h ^= ((uint64_t)(data2[4])) << 32;
-		case 4: h ^= ((uint64_t)(data2[3])) << 24;
-		case 3: h ^= ((uint64_t)(data2[2])) << 16;
-		case 2: h ^= ((uint64_t)(data2[1])) << 8;
-		case 1: h ^= ((uint64_t)(data2[0]));
-			h *= m;
-	}
+JEMALLOC_INLINE uint64_t
+hash_get_block_64(const uint64_t *p, int i)
+{
 
-	h ^= h >> r;
-	h *= m;
-	h ^= h >> r;
+	return (p[i]);
+}
+
+JEMALLOC_INLINE uint32_t
+hash_fmix_32(uint32_t h)
+{
+
+	h ^= h >> 16;
+	h *= 0x85ebca6b;
+	h ^= h >> 13;
+	h *= 0xc2b2ae35;
+	h ^= h >> 16;
 
 	return (h);
 }
+
+JEMALLOC_INLINE uint64_t
+hash_fmix_64(uint64_t k)
+{
+
+	k ^= k >> 33;
+	k *= QU(0xff51afd7ed558ccdLLU);
+	k ^= k >> 33;
+	k *= QU(0xc4ceb9fe1a85ec53LLU);
+	k ^= k >> 33;
+
+	return (k);
+}
+
+JEMALLOC_INLINE uint32_t
+hash_x86_32(const void *key, int len, uint32_t seed)
+{
+	const uint8_t *data = (const uint8_t *) key;
+	const int nblocks = len / 4;
+
+	uint32_t h1 = seed;
+
+	const uint32_t c1 = 0xcc9e2d51;
+	const uint32_t c2 = 0x1b873593;
+
+	/* body */
+	{
+		const uint32_t *blocks = (const uint32_t *) (data + nblocks*4);
+		int i;
+
+		for (i = -nblocks; i; i++) {
+			uint32_t k1 = hash_get_block_32(blocks, i);
+
+			k1 *= c1;
+			k1 = hash_rotl_32(k1, 15);
+			k1 *= c2;
+
+			h1 ^= k1;
+			h1 = hash_rotl_32(h1, 13);
+			h1 = h1*5 + 0xe6546b64;
+		}
+	}
+
+	/* tail */
+	{
+		const uint8_t *tail = (const uint8_t *) (data + nblocks*4);
+
+		uint32_t k1 = 0;
+
+		switch (len & 3) {
+		case 3: k1 ^= tail[2] << 16;
+		case 2: k1 ^= tail[1] << 8;
+		case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15);
+			k1 *= c2; h1 ^= k1;
+		}
+	}
+
+	/* finalization */
+	h1 ^= len;
+
+	h1 = hash_fmix_32(h1);
+
+	return (h1);
+}
+
+UNUSED JEMALLOC_INLINE void
+hash_x86_128(const void *key, const int len, uint32_t seed,
+    uint64_t r_out[2])
+{
+	const uint8_t * data = (const uint8_t *) key;
+	const int nblocks = len / 16;
+
+	uint32_t h1 = seed;
+	uint32_t h2 = seed;
+	uint32_t h3 = seed;
+	uint32_t h4 = seed;
+
+	const uint32_t c1 = 0x239b961b;
+	const uint32_t c2 = 0xab0e9789;
+	const uint32_t c3 = 0x38b34ae5;
+	const uint32_t c4 = 0xa1e38b93;
+
+	/* body */
+	{
+		const uint32_t *blocks = (const uint32_t *) (data + nblocks*16);
+		int i;
+
+		for (i = -nblocks; i; i++) {
+			uint32_t k1 = hash_get_block_32(blocks, i*4 + 0);
+			uint32_t k2 = hash_get_block_32(blocks, i*4 + 1);
+			uint32_t k3 = hash_get_block_32(blocks, i*4 + 2);
+			uint32_t k4 = hash_get_block_32(blocks, i*4 + 3);
+
+			k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1;
+
+			h1 = hash_rotl_32(h1, 19); h1 += h2;
+			h1 = h1*5 + 0x561ccd1b;
+
+			k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2;
+
+			h2 = hash_rotl_32(h2, 17); h2 += h3;
+			h2 = h2*5 + 0x0bcaa747;
+
+			k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3;
+
+			h3 = hash_rotl_32(h3, 15); h3 += h4;
+			h3 = h3*5 + 0x96cd1c35;
+
+			k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4;
+
+			h4 = hash_rotl_32(h4, 13); h4 += h1;
+			h4 = h4*5 + 0x32ac3b17;
+		}
+	}
+
+	/* tail */
+	{
+		const uint8_t *tail = (const uint8_t *) (data + nblocks*16);
+		uint32_t k1 = 0;
+		uint32_t k2 = 0;
+		uint32_t k3 = 0;
+		uint32_t k4 = 0;
+
+		switch (len & 15) {
+		case 15: k4 ^= tail[14] << 16;
+		case 14: k4 ^= tail[13] << 8;
+		case 13: k4 ^= tail[12] << 0;
+			k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4;
+
+		case 12: k3 ^= tail[11] << 24;
+		case 11: k3 ^= tail[10] << 16;
+		case 10: k3 ^= tail[ 9] << 8;
+		case  9: k3 ^= tail[ 8] << 0;
+		     k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3;
+
+		case  8: k2 ^= tail[ 7] << 24;
+		case  7: k2 ^= tail[ 6] << 16;
+		case  6: k2 ^= tail[ 5] << 8;
+		case  5: k2 ^= tail[ 4] << 0;
+			k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2;
+
+		case  4: k1 ^= tail[ 3] << 24;
+		case  3: k1 ^= tail[ 2] << 16;
+		case  2: k1 ^= tail[ 1] << 8;
+		case  1: k1 ^= tail[ 0] << 0;
+			k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1;
+		}
+	}
+
+	/* finalization */
+	h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len;
+
+	h1 += h2; h1 += h3; h1 += h4;
+	h2 += h1; h3 += h1; h4 += h1;
+
+	h1 = hash_fmix_32(h1);
+	h2 = hash_fmix_32(h2);
+	h3 = hash_fmix_32(h3);
+	h4 = hash_fmix_32(h4);
+
+	h1 += h2; h1 += h3; h1 += h4;
+	h2 += h1; h3 += h1; h4 += h1;
+
+	r_out[0] = (((uint64_t) h2) << 32) | h1;
+	r_out[1] = (((uint64_t) h4) << 32) | h3;
+}
+
+UNUSED JEMALLOC_INLINE void
+hash_x64_128(const void *key, const int len, const uint32_t seed,
+    uint64_t r_out[2])
+{
+	const uint8_t *data = (const uint8_t *) key;
+	const int nblocks = len / 16;
+
+	uint64_t h1 = seed;
+	uint64_t h2 = seed;
+
+	const uint64_t c1 = QU(0x87c37b91114253d5LLU);
+	const uint64_t c2 = QU(0x4cf5ad432745937fLLU);
+
+	/* body */
+	{
+		const uint64_t *blocks = (const uint64_t *) (data);
+		int i;
+
+		for (i = 0; i < nblocks; i++) {
+			uint64_t k1 = hash_get_block_64(blocks, i*2 + 0);
+			uint64_t k2 = hash_get_block_64(blocks, i*2 + 1);
+
+			k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1;
+
+			h1 = hash_rotl_64(h1, 27); h1 += h2;
+			h1 = h1*5 + 0x52dce729;
+
+			k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2;
+
+			h2 = hash_rotl_64(h2, 31); h2 += h1;
+			h2 = h2*5 + 0x38495ab5;
+		}
+	}
+
+	/* tail */
+	{
+		const uint8_t *tail = (const uint8_t*)(data + nblocks*16);
+		uint64_t k1 = 0;
+		uint64_t k2 = 0;
+
+		switch (len & 15) {
+		case 15: k2 ^= ((uint64_t)(tail[14])) << 48;
+		case 14: k2 ^= ((uint64_t)(tail[13])) << 40;
+		case 13: k2 ^= ((uint64_t)(tail[12])) << 32;
+		case 12: k2 ^= ((uint64_t)(tail[11])) << 24;
+		case 11: k2 ^= ((uint64_t)(tail[10])) << 16;
+		case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8;
+		case  9: k2 ^= ((uint64_t)(tail[ 8])) << 0;
+			k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2;
+
+		case  8: k1 ^= ((uint64_t)(tail[ 7])) << 56;
+		case  7: k1 ^= ((uint64_t)(tail[ 6])) << 48;
+		case  6: k1 ^= ((uint64_t)(tail[ 5])) << 40;
+		case  5: k1 ^= ((uint64_t)(tail[ 4])) << 32;
+		case  4: k1 ^= ((uint64_t)(tail[ 3])) << 24;
+		case  3: k1 ^= ((uint64_t)(tail[ 2])) << 16;
+		case  2: k1 ^= ((uint64_t)(tail[ 1])) << 8;
+		case  1: k1 ^= ((uint64_t)(tail[ 0])) << 0;
+			k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1;
+		}
+	}
+
+	/* finalization */
+	h1 ^= len; h2 ^= len;
+
+	h1 += h2;
+	h2 += h1;
+
+	h1 = hash_fmix_64(h1);
+	h2 = hash_fmix_64(h2);
+
+	h1 += h2;
+	h2 += h1;
+
+	r_out[0] = h1;
+	r_out[1] = h2;
+}
+
+/******************************************************************************/
+/* API. */
+JEMALLOC_INLINE void
+hash(const void *key, size_t len, const uint32_t seed, size_t r_hash[2])
+{
+#if (LG_SIZEOF_PTR == 3)
+	hash_x64_128(key, len, seed, (uint64_t *)r_hash);
+#else
+	uint64_t hashes[2];
+	hash_x86_128(key, len, seed, hashes);
+	r_hash[0] = (size_t)hashes[0];
+	r_hash[1] = (size_t)hashes[1];
+#endif
+}
 #endif
 
 #endif /* JEMALLOC_H_INLINES */

include/jemalloc/internal/huge.h

@@ -9,29 +9,32 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-#ifdef JEMALLOC_STATS
 /* Huge allocation statistics. */
 extern uint64_t		huge_nmalloc;
 extern uint64_t		huge_ndalloc;
 extern size_t		huge_allocated;
-#endif
 
 /* Protects chunk-related data structures. */
 extern malloc_mutex_t	huge_mtx;
 
 void	*huge_malloc(size_t size, bool zero);
 void	*huge_palloc(size_t size, size_t alignment, bool zero);
-void	*huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
+bool	huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
     size_t extra);
 void	*huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
-    size_t alignment, bool zero);
+    size_t alignment, bool zero, bool try_tcache_dalloc);
+#ifdef JEMALLOC_JET
+typedef void (huge_dalloc_junk_t)(void *, size_t);
+extern huge_dalloc_junk_t *huge_dalloc_junk;
+#endif
 void	huge_dalloc(void *ptr, bool unmap);
 size_t	huge_salloc(const void *ptr);
-#ifdef JEMALLOC_PROF
 prof_ctx_t	*huge_prof_ctx_get(const void *ptr);
 void	huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
-#endif
 bool	huge_boot(void);
+void	huge_prefork(void);
+void	huge_postfork_parent(void);
+void	huge_postfork_child(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -0,0 +1,199 @@
+#ifndef JEMALLOC_INTERNAL_DEFS_H_
+#define	JEMALLOC_INTERNAL_DEFS_H_
+/*
+ * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all
+ * public APIs to be prefixed.  This makes it possible, with some care, to use
+ * multiple allocators simultaneously.
+ */
+#undef JEMALLOC_PREFIX
+#undef JEMALLOC_CPREFIX
+
+/*
+ * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs.
+ * For shared libraries, symbol visibility mechanisms prevent these symbols
+ * from being exported, but for static libraries, naming collisions are a real
+ * possibility.
+ */
+#undef JEMALLOC_PRIVATE_NAMESPACE
+
+/*
+ * Hyper-threaded CPUs may need a special instruction inside spin loops in
+ * order to yield to another virtual CPU.
+ */
+#undef CPU_SPINWAIT
+
+/* Defined if the equivalent of FreeBSD's atomic(9) functions are available. */
+#undef JEMALLOC_ATOMIC9
+
+/*
+ * Defined if OSAtomic*() functions are available, as provided by Darwin, and
+ * documented in the atomic(3) manual page.
+ */
+#undef JEMALLOC_OSATOMIC
+
+/*
+ * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and
+ * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite
+ * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the
+ * functions are defined in libgcc instead of being inlines)
+ */
+#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4
+
+/*
+ * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and
+ * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite
+ * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the
+ * functions are defined in libgcc instead of being inlines)
+ */
+#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8
+
+/*
+ * Defined if OSSpin*() functions are available, as provided by Darwin, and
+ * documented in the spinlock(3) manual page.
+ */
+#undef JEMALLOC_OSSPIN
+
+/*
+ * Defined if _malloc_thread_cleanup() exists.  At least in the case of
+ * FreeBSD, pthread_key_create() allocates, which if used during malloc
+ * bootstrapping will cause recursion into the pthreads library.  Therefore, if
+ * _malloc_thread_cleanup() exists, use it as the basis for thread cleanup in
+ * malloc_tsd.
+ */
+#undef JEMALLOC_MALLOC_THREAD_CLEANUP
+
+/*
+ * Defined if threaded initialization is known to be safe on this platform.
+ * Among other things, it must be possible to initialize a mutex without
+ * triggering allocation in order for threaded allocation to be safe.
+ */
+#undef JEMALLOC_THREADED_INIT
+
+/*
+ * Defined if the pthreads implementation defines
+ * _pthread_mutex_init_calloc_cb(), in which case the function is used in order
+ * to avoid recursive allocation during mutex initialization.
+ */
+#undef JEMALLOC_MUTEX_INIT_CB
+
+/* Defined if sbrk() is supported. */
+#undef JEMALLOC_HAVE_SBRK
+
+/* Non-empty if the tls_model attribute is supported. */
+#undef JEMALLOC_TLS_MODEL
+
+/* JEMALLOC_CC_SILENCE enables code that silences unuseful compiler warnings. */
+#undef JEMALLOC_CC_SILENCE
+
+/* JEMALLOC_CODE_COVERAGE enables test code coverage analysis. */
+#undef JEMALLOC_CODE_COVERAGE
+
+/*
+ * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables
+ * inline functions.
+ */
+#undef JEMALLOC_DEBUG
+
+/* JEMALLOC_STATS enables statistics calculation. */
+#undef JEMALLOC_STATS
+
+/* JEMALLOC_PROF enables allocation profiling. */
+#undef JEMALLOC_PROF
+
+/* Use libunwind for profile backtracing if defined. */
+#undef JEMALLOC_PROF_LIBUNWIND
+
+/* Use libgcc for profile backtracing if defined. */
+#undef JEMALLOC_PROF_LIBGCC
+
+/* Use gcc intrinsics for profile backtracing if defined. */
+#undef JEMALLOC_PROF_GCC
+
+/*
+ * JEMALLOC_TCACHE enables a thread-specific caching layer for small objects.
+ * This makes it possible to allocate/deallocate objects without any locking
+ * when the cache is in the steady state.
+ */
+#undef JEMALLOC_TCACHE
+
+/*
+ * JEMALLOC_DSS enables use of sbrk(2) to allocate chunks from the data storage
+ * segment (DSS).
+ */
+#undef JEMALLOC_DSS
+
+/* Support memory filling (junk/zero/quarantine/redzone). */
+#undef JEMALLOC_FILL
+
+/* Support utrace(2)-based tracing. */
+#undef JEMALLOC_UTRACE
+
+/* Support Valgrind. */
+#undef JEMALLOC_VALGRIND
+
+/* Support optional abort() on OOM. */
+#undef JEMALLOC_XMALLOC
+
+/* Support lazy locking (avoid locking unless a second thread is launched). */
+#undef JEMALLOC_LAZY_LOCK
+
+/* One page is 2^STATIC_PAGE_SHIFT bytes. */
+#undef STATIC_PAGE_SHIFT
+
+/*
+ * If defined, use munmap() to unmap freed chunks, rather than storing them for
+ * later reuse.  This is disabled by default on Linux because common sequences
+ * of mmap()/munmap() calls will cause virtual memory map holes.
+ */
+#undef JEMALLOC_MUNMAP
+
+/*
+ * If defined, use mremap(...MREMAP_FIXED...) for huge realloc().  This is
+ * disabled by default because it is Linux-specific and it will cause virtual
+ * memory map holes, much like munmap(2) does.
+ */
+#undef JEMALLOC_MREMAP
+
+/* TLS is used to map arenas and magazine caches to threads. */
+#undef JEMALLOC_TLS
+
+/*
+ * JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside
+ * within jemalloc-owned chunks before dereferencing them.
+ */
+#undef JEMALLOC_IVSALLOC
+
+/*
+ * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
+ */
+#undef JEMALLOC_ZONE
+#undef JEMALLOC_ZONE_VERSION
+
+/*
+ * Methods for purging unused pages differ between operating systems.
+ *
+ *   madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages,
+ *                                 such that new pages will be demand-zeroed if
+ *                                 the address region is later touched.
+ *   madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being
+ *                             unused, such that they will be discarded rather
+ *                             than swapped out.
+ */
+#undef JEMALLOC_PURGE_MADVISE_DONTNEED
+#undef JEMALLOC_PURGE_MADVISE_FREE
+
+/*
+ * Define if operating system has alloca.h header.
+ */
+#undef JEMALLOC_HAS_ALLOCA_H
+
+/* sizeof(int) == 2^LG_SIZEOF_INT. */
+#undef LG_SIZEOF_INT
+
+/* sizeof(long) == 2^LG_SIZEOF_LONG. */
+#undef LG_SIZEOF_LONG
+
+/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */
+#undef LG_SIZEOF_INTMAX_T
+
+#endif /* JEMALLOC_INTERNAL_DEFS_H_ */

include/jemalloc/internal/jemalloc_internal_macros.h

@@ -0,0 +1,47 @@
+/*
+ * JEMALLOC_ALWAYS_INLINE and JEMALLOC_INLINE are used within header files for
+ * functions that are static inline functions if inlining is enabled, and
+ * single-definition library-private functions if inlining is disabled.
+ *
+ * JEMALLOC_ALWAYS_INLINE_C and JEMALLOC_INLINE_C are for use in .c files, in
+ * which case the denoted functions are always static, regardless of whether
+ * inlining is enabled.
+ */
+#if defined(JEMALLOC_DEBUG) || defined(JEMALLOC_CODE_COVERAGE)
+   /* Disable inlining to make debugging/profiling easier. */
+#  define JEMALLOC_ALWAYS_INLINE
+#  define JEMALLOC_ALWAYS_INLINE_C static
+#  define JEMALLOC_INLINE
+#  define JEMALLOC_INLINE_C static
+#  define inline
+#else
+#  define JEMALLOC_ENABLE_INLINE
+#  ifdef JEMALLOC_HAVE_ATTR
+#    define JEMALLOC_ALWAYS_INLINE \
+	 static inline JEMALLOC_ATTR(unused) JEMALLOC_ATTR(always_inline)
+#    define JEMALLOC_ALWAYS_INLINE_C \
+	 static inline JEMALLOC_ATTR(always_inline)
+#  else
+#    define JEMALLOC_ALWAYS_INLINE static inline
+#    define JEMALLOC_ALWAYS_INLINE_C static inline
+#  endif
+#  define JEMALLOC_INLINE static inline
+#  define JEMALLOC_INLINE_C static inline
+#  ifdef _MSC_VER
+#    define inline _inline
+#  endif
+#endif
+
+#ifdef JEMALLOC_CC_SILENCE
+#  define UNUSED JEMALLOC_ATTR(unused)
+#else
+#  define UNUSED
+#endif
+
+#define	ZU(z)	((size_t)z)
+#define	QU(q)	((uint64_t)q)
+#define	QI(q)	((int64_t)q)
+
+#ifndef __DECONST
+#  define	__DECONST(type, var)	((type)(uintptr_t)(const void *)(var))
+#endif

include/jemalloc/internal/mb.h

@@ -54,7 +54,7 @@ mb_write(void)
 	    );
 #endif
 }
-#elif (defined(__amd64_) || defined(__x86_64__))
+#elif (defined(__amd64__) || defined(__x86_64__))
 JEMALLOC_INLINE void
 mb_write(void)
 {
@@ -87,6 +87,13 @@ mb_write(void)
 	    : "memory" /* Clobbers. */
 	    );
 }
+#elif defined(__tile__)
+JEMALLOC_INLINE void
+mb_write(void)
+{
+
+	__sync_synchronize();
+}
 #else
 /*
  * This is much slower than a simple memory barrier, but the semantics of mutex

include/jemalloc/internal/mutex.h

@@ -1,22 +1,42 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_TYPES
 
-#ifdef JEMALLOC_OSSPIN
-typedef OSSpinLock malloc_mutex_t;
-#else
-typedef pthread_mutex_t malloc_mutex_t;
-#endif
+typedef struct malloc_mutex_s malloc_mutex_t;
 
-#ifdef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP
-#  define MALLOC_MUTEX_INITIALIZER PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP
+#ifdef _WIN32
+#  define MALLOC_MUTEX_INITIALIZER
+#elif (defined(JEMALLOC_OSSPIN))
+#  define MALLOC_MUTEX_INITIALIZER {0}
+#elif (defined(JEMALLOC_MUTEX_INIT_CB))
+#  define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER, NULL}
 #else
-#  define MALLOC_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+#  if (defined(PTHREAD_MUTEX_ADAPTIVE_NP) &&				\
+       defined(PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP))
+#    define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_ADAPTIVE_NP
+#    define MALLOC_MUTEX_INITIALIZER {PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP}
+#  else
+#    define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
+#    define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER}
+#  endif
 #endif
 
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
 
+struct malloc_mutex_s {
+#ifdef _WIN32
+	CRITICAL_SECTION	lock;
+#elif (defined(JEMALLOC_OSSPIN))
+	OSSpinLock		lock;
+#elif (defined(JEMALLOC_MUTEX_INIT_CB))
+	pthread_mutex_t		lock;
+	malloc_mutex_t		*postponed_next;
+#else
+	pthread_mutex_t		lock;
+#endif
+};
+
 #endif /* JEMALLOC_H_STRUCTS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
@@ -24,11 +44,15 @@ typedef pthread_mutex_t malloc_mutex_t;
 #ifdef JEMALLOC_LAZY_LOCK
 extern bool isthreaded;
 #else
+#  undef isthreaded /* Undo private_namespace.h definition. */
 #  define isthreaded true
 #endif
 
 bool	malloc_mutex_init(malloc_mutex_t *mutex);
-void	malloc_mutex_destroy(malloc_mutex_t *mutex);
+void	malloc_mutex_prefork(malloc_mutex_t *mutex);
+void	malloc_mutex_postfork_parent(malloc_mutex_t *mutex);
+void	malloc_mutex_postfork_child(malloc_mutex_t *mutex);
+bool	mutex_boot(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
@@ -36,7 +60,6 @@ void	malloc_mutex_destroy(malloc_mutex_t *mutex);
 
 #ifndef JEMALLOC_ENABLE_INLINE
 void	malloc_mutex_lock(malloc_mutex_t *mutex);
-bool	malloc_mutex_trylock(malloc_mutex_t *mutex);
 void	malloc_mutex_unlock(malloc_mutex_t *mutex);
 #endif
 
@@ -46,37 +69,27 @@ malloc_mutex_lock(malloc_mutex_t *mutex)
 {
 
 	if (isthreaded) {
-#ifdef JEMALLOC_OSSPIN
-		OSSpinLockLock(mutex);
+#ifdef _WIN32
+		EnterCriticalSection(&mutex->lock);
+#elif (defined(JEMALLOC_OSSPIN))
+		OSSpinLockLock(&mutex->lock);
 #else
-		pthread_mutex_lock(mutex);
+		pthread_mutex_lock(&mutex->lock);
 #endif
 	}
 }
 
-JEMALLOC_INLINE bool
-malloc_mutex_trylock(malloc_mutex_t *mutex)
-{
-
-	if (isthreaded) {
-#ifdef JEMALLOC_OSSPIN
-		return (OSSpinLockTry(mutex) == false);
-#else
-		return (pthread_mutex_trylock(mutex) != 0);
-#endif
-	} else
-		return (false);
-}
-
 JEMALLOC_INLINE void
 malloc_mutex_unlock(malloc_mutex_t *mutex)
 {
 
 	if (isthreaded) {
-#ifdef JEMALLOC_OSSPIN
-		OSSpinLockUnlock(mutex);
+#ifdef _WIN32
+		LeaveCriticalSection(&mutex->lock);
+#elif (defined(JEMALLOC_OSSPIN))
+		OSSpinLockUnlock(&mutex->lock);
 #else
-		pthread_mutex_unlock(mutex);
+		pthread_mutex_unlock(&mutex->lock);
 #endif
 	}
 }

include/jemalloc/internal/private_namespace.h

@@ -1,195 +0,0 @@
-#define	arena_bin_index JEMALLOC_N(arena_bin_index)
-#define	arena_boot JEMALLOC_N(arena_boot)
-#define	arena_dalloc JEMALLOC_N(arena_dalloc)
-#define	arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin)
-#define	arena_dalloc_large JEMALLOC_N(arena_dalloc_large)
-#define	arena_malloc JEMALLOC_N(arena_malloc)
-#define	arena_malloc_large JEMALLOC_N(arena_malloc_large)
-#define	arena_malloc_small JEMALLOC_N(arena_malloc_small)
-#define	arena_new JEMALLOC_N(arena_new)
-#define	arena_palloc JEMALLOC_N(arena_palloc)
-#define	arena_prof_accum JEMALLOC_N(arena_prof_accum)
-#define	arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get)
-#define	arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set)
-#define	arena_prof_promoted JEMALLOC_N(arena_prof_promoted)
-#define	arena_purge_all JEMALLOC_N(arena_purge_all)
-#define	arena_ralloc JEMALLOC_N(arena_ralloc)
-#define	arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move)
-#define	arena_run_regind JEMALLOC_N(arena_run_regind)
-#define	arena_salloc JEMALLOC_N(arena_salloc)
-#define	arena_salloc_demote JEMALLOC_N(arena_salloc_demote)
-#define	arena_stats_merge JEMALLOC_N(arena_stats_merge)
-#define	arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small)
-#define	arenas_bin_i_index JEMALLOC_N(arenas_bin_i_index)
-#define	arenas_extend JEMALLOC_N(arenas_extend)
-#define	arenas_lrun_i_index JEMALLOC_N(arenas_lrun_i_index)
-#define	atomic_add_uint32 JEMALLOC_N(atomic_add_uint32)
-#define	atomic_add_uint64 JEMALLOC_N(atomic_add_uint64)
-#define	atomic_sub_uint32 JEMALLOC_N(atomic_sub_uint32)
-#define	atomic_sub_uint64 JEMALLOC_N(atomic_sub_uint64)
-#define	base_alloc JEMALLOC_N(base_alloc)
-#define	base_boot JEMALLOC_N(base_boot)
-#define	base_node_alloc JEMALLOC_N(base_node_alloc)
-#define	base_node_dealloc JEMALLOC_N(base_node_dealloc)
-#define	bitmap_full JEMALLOC_N(bitmap_full)
-#define	bitmap_get JEMALLOC_N(bitmap_get)
-#define	bitmap_info_init JEMALLOC_N(bitmap_info_init)
-#define	bitmap_info_ngroups JEMALLOC_N(bitmap_info_ngroups)
-#define	bitmap_init JEMALLOC_N(bitmap_init)
-#define	bitmap_set JEMALLOC_N(bitmap_set)
-#define	bitmap_sfu JEMALLOC_N(bitmap_sfu)
-#define	bitmap_size JEMALLOC_N(bitmap_size)
-#define	bitmap_unset JEMALLOC_N(bitmap_unset)
-#define	bt_init JEMALLOC_N(bt_init)
-#define	buferror JEMALLOC_N(buferror)
-#define	choose_arena JEMALLOC_N(choose_arena)
-#define	choose_arena_hard JEMALLOC_N(choose_arena_hard)
-#define	chunk_alloc JEMALLOC_N(chunk_alloc)
-#define	chunk_alloc_dss JEMALLOC_N(chunk_alloc_dss)
-#define	chunk_alloc_mmap JEMALLOC_N(chunk_alloc_mmap)
-#define	chunk_alloc_mmap_noreserve JEMALLOC_N(chunk_alloc_mmap_noreserve)
-#define	chunk_alloc_swap JEMALLOC_N(chunk_alloc_swap)
-#define	chunk_boot JEMALLOC_N(chunk_boot)
-#define	chunk_dealloc JEMALLOC_N(chunk_dealloc)
-#define	chunk_dealloc_dss JEMALLOC_N(chunk_dealloc_dss)
-#define	chunk_dealloc_mmap JEMALLOC_N(chunk_dealloc_mmap)
-#define	chunk_dealloc_swap JEMALLOC_N(chunk_dealloc_swap)
-#define	chunk_dss_boot JEMALLOC_N(chunk_dss_boot)
-#define	chunk_in_dss JEMALLOC_N(chunk_in_dss)
-#define	chunk_in_swap JEMALLOC_N(chunk_in_swap)
-#define	chunk_mmap_boot JEMALLOC_N(chunk_mmap_boot)
-#define	chunk_swap_boot JEMALLOC_N(chunk_swap_boot)
-#define	chunk_swap_enable JEMALLOC_N(chunk_swap_enable)
-#define	ckh_bucket_search JEMALLOC_N(ckh_bucket_search)
-#define	ckh_count JEMALLOC_N(ckh_count)
-#define	ckh_delete JEMALLOC_N(ckh_delete)
-#define	ckh_evict_reloc_insert JEMALLOC_N(ckh_evict_reloc_insert)
-#define	ckh_insert JEMALLOC_N(ckh_insert)
-#define	ckh_isearch JEMALLOC_N(ckh_isearch)
-#define	ckh_iter JEMALLOC_N(ckh_iter)
-#define	ckh_new JEMALLOC_N(ckh_new)
-#define	ckh_pointer_hash JEMALLOC_N(ckh_pointer_hash)
-#define	ckh_pointer_keycomp JEMALLOC_N(ckh_pointer_keycomp)
-#define	ckh_rebuild JEMALLOC_N(ckh_rebuild)
-#define	ckh_remove JEMALLOC_N(ckh_remove)
-#define	ckh_search JEMALLOC_N(ckh_search)
-#define	ckh_string_hash JEMALLOC_N(ckh_string_hash)
-#define	ckh_string_keycomp JEMALLOC_N(ckh_string_keycomp)
-#define	ckh_try_bucket_insert JEMALLOC_N(ckh_try_bucket_insert)
-#define	ckh_try_insert JEMALLOC_N(ckh_try_insert)
-#define	create_zone JEMALLOC_N(create_zone)
-#define	ctl_boot JEMALLOC_N(ctl_boot)
-#define	ctl_bymib JEMALLOC_N(ctl_bymib)
-#define	ctl_byname JEMALLOC_N(ctl_byname)
-#define	ctl_nametomib JEMALLOC_N(ctl_nametomib)
-#define	extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first)
-#define	extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert)
-#define	extent_tree_ad_iter JEMALLOC_N(extent_tree_ad_iter)
-#define	extent_tree_ad_iter_recurse JEMALLOC_N(extent_tree_ad_iter_recurse)
-#define	extent_tree_ad_iter_start JEMALLOC_N(extent_tree_ad_iter_start)
-#define	extent_tree_ad_last JEMALLOC_N(extent_tree_ad_last)
-#define	extent_tree_ad_new JEMALLOC_N(extent_tree_ad_new)
-#define	extent_tree_ad_next JEMALLOC_N(extent_tree_ad_next)
-#define	extent_tree_ad_nsearch JEMALLOC_N(extent_tree_ad_nsearch)
-#define	extent_tree_ad_prev JEMALLOC_N(extent_tree_ad_prev)
-#define	extent_tree_ad_psearch JEMALLOC_N(extent_tree_ad_psearch)
-#define	extent_tree_ad_remove JEMALLOC_N(extent_tree_ad_remove)
-#define	extent_tree_ad_reverse_iter JEMALLOC_N(extent_tree_ad_reverse_iter)
-#define	extent_tree_ad_reverse_iter_recurse JEMALLOC_N(extent_tree_ad_reverse_iter_recurse)
-#define	extent_tree_ad_reverse_iter_start JEMALLOC_N(extent_tree_ad_reverse_iter_start)
-#define	extent_tree_ad_search JEMALLOC_N(extent_tree_ad_search)
-#define	extent_tree_szad_first JEMALLOC_N(extent_tree_szad_first)
-#define	extent_tree_szad_insert JEMALLOC_N(extent_tree_szad_insert)
-#define	extent_tree_szad_iter JEMALLOC_N(extent_tree_szad_iter)
-#define	extent_tree_szad_iter_recurse JEMALLOC_N(extent_tree_szad_iter_recurse)
-#define	extent_tree_szad_iter_start JEMALLOC_N(extent_tree_szad_iter_start)
-#define	extent_tree_szad_last JEMALLOC_N(extent_tree_szad_last)
-#define	extent_tree_szad_new JEMALLOC_N(extent_tree_szad_new)
-#define	extent_tree_szad_next JEMALLOC_N(extent_tree_szad_next)
-#define	extent_tree_szad_nsearch JEMALLOC_N(extent_tree_szad_nsearch)
-#define	extent_tree_szad_prev JEMALLOC_N(extent_tree_szad_prev)
-#define	extent_tree_szad_psearch JEMALLOC_N(extent_tree_szad_psearch)
-#define	extent_tree_szad_remove JEMALLOC_N(extent_tree_szad_remove)
-#define	extent_tree_szad_reverse_iter JEMALLOC_N(extent_tree_szad_reverse_iter)
-#define	extent_tree_szad_reverse_iter_recurse JEMALLOC_N(extent_tree_szad_reverse_iter_recurse)
-#define	extent_tree_szad_reverse_iter_start JEMALLOC_N(extent_tree_szad_reverse_iter_start)
-#define	extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search)
-#define	hash JEMALLOC_N(hash)
-#define	huge_boot JEMALLOC_N(huge_boot)
-#define	huge_dalloc JEMALLOC_N(huge_dalloc)
-#define	huge_malloc JEMALLOC_N(huge_malloc)
-#define	huge_palloc JEMALLOC_N(huge_palloc)
-#define	huge_prof_ctx_get JEMALLOC_N(huge_prof_ctx_get)
-#define	huge_prof_ctx_set JEMALLOC_N(huge_prof_ctx_set)
-#define	huge_ralloc JEMALLOC_N(huge_ralloc)
-#define	huge_ralloc_no_move JEMALLOC_N(huge_ralloc_no_move)
-#define	huge_salloc JEMALLOC_N(huge_salloc)
-#define	iallocm JEMALLOC_N(iallocm)
-#define	icalloc JEMALLOC_N(icalloc)
-#define	idalloc JEMALLOC_N(idalloc)
-#define	imalloc JEMALLOC_N(imalloc)
-#define	ipalloc JEMALLOC_N(ipalloc)
-#define	iralloc JEMALLOC_N(iralloc)
-#define	isalloc JEMALLOC_N(isalloc)
-#define	ivsalloc JEMALLOC_N(ivsalloc)
-#define	jemalloc_darwin_init JEMALLOC_N(jemalloc_darwin_init)
-#define	jemalloc_postfork JEMALLOC_N(jemalloc_postfork)
-#define	jemalloc_prefork JEMALLOC_N(jemalloc_prefork)
-#define	malloc_cprintf JEMALLOC_N(malloc_cprintf)
-#define	malloc_mutex_destroy JEMALLOC_N(malloc_mutex_destroy)
-#define	malloc_mutex_init JEMALLOC_N(malloc_mutex_init)
-#define	malloc_mutex_lock JEMALLOC_N(malloc_mutex_lock)
-#define	malloc_mutex_trylock JEMALLOC_N(malloc_mutex_trylock)
-#define	malloc_mutex_unlock JEMALLOC_N(malloc_mutex_unlock)
-#define	malloc_printf JEMALLOC_N(malloc_printf)
-#define	malloc_write JEMALLOC_N(malloc_write)
-#define	mb_write JEMALLOC_N(mb_write)
-#define	pow2_ceil JEMALLOC_N(pow2_ceil)
-#define	prof_backtrace JEMALLOC_N(prof_backtrace)
-#define	prof_boot0 JEMALLOC_N(prof_boot0)
-#define	prof_boot1 JEMALLOC_N(prof_boot1)
-#define	prof_boot2 JEMALLOC_N(prof_boot2)
-#define	prof_ctx_get JEMALLOC_N(prof_ctx_get)
-#define	prof_ctx_set JEMALLOC_N(prof_ctx_set)
-#define	prof_free JEMALLOC_N(prof_free)
-#define	prof_gdump JEMALLOC_N(prof_gdump)
-#define	prof_idump JEMALLOC_N(prof_idump)
-#define	prof_lookup JEMALLOC_N(prof_lookup)
-#define	prof_malloc JEMALLOC_N(prof_malloc)
-#define	prof_mdump JEMALLOC_N(prof_mdump)
-#define	prof_realloc JEMALLOC_N(prof_realloc)
-#define	prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update)
-#define	prof_sample_threshold_update JEMALLOC_N(prof_sample_threshold_update)
-#define	prof_tdata_init JEMALLOC_N(prof_tdata_init)
-#define	pthread_create JEMALLOC_N(pthread_create)
-#define	rtree_get JEMALLOC_N(rtree_get)
-#define	rtree_get_locked JEMALLOC_N(rtree_get_locked)
-#define	rtree_new JEMALLOC_N(rtree_new)
-#define	rtree_set JEMALLOC_N(rtree_set)
-#define	s2u JEMALLOC_N(s2u)
-#define	sa2u JEMALLOC_N(sa2u)
-#define	stats_arenas_i_bins_j_index JEMALLOC_N(stats_arenas_i_bins_j_index)
-#define	stats_arenas_i_index JEMALLOC_N(stats_arenas_i_index)
-#define	stats_arenas_i_lruns_j_index JEMALLOC_N(stats_arenas_i_lruns_j_index)
-#define	stats_cactive_add JEMALLOC_N(stats_cactive_add)
-#define	stats_cactive_get JEMALLOC_N(stats_cactive_get)
-#define	stats_cactive_sub JEMALLOC_N(stats_cactive_sub)
-#define	stats_print JEMALLOC_N(stats_print)
-#define	szone2ozone JEMALLOC_N(szone2ozone)
-#define	tcache_alloc_easy JEMALLOC_N(tcache_alloc_easy)
-#define	tcache_alloc_large JEMALLOC_N(tcache_alloc_large)
-#define	tcache_alloc_small JEMALLOC_N(tcache_alloc_small)
-#define	tcache_alloc_small_hard JEMALLOC_N(tcache_alloc_small_hard)
-#define	tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large)
-#define	tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small)
-#define	tcache_boot JEMALLOC_N(tcache_boot)
-#define	tcache_create JEMALLOC_N(tcache_create)
-#define	tcache_dalloc_large JEMALLOC_N(tcache_dalloc_large)
-#define	tcache_dalloc_small JEMALLOC_N(tcache_dalloc_small)
-#define	tcache_destroy JEMALLOC_N(tcache_destroy)
-#define	tcache_event JEMALLOC_N(tcache_event)
-#define	tcache_get JEMALLOC_N(tcache_get)
-#define	tcache_stats_merge JEMALLOC_N(tcache_stats_merge)
-#define	thread_allocated_get JEMALLOC_N(thread_allocated_get)
-#define	thread_allocated_get_hard JEMALLOC_N(thread_allocated_get_hard)
-#define	u2s JEMALLOC_N(u2s)

include/jemalloc/internal/private_namespace.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+for symbol in `cat $1` ; do
+  echo "#define	${symbol} JEMALLOC_N(${symbol})"
+done

include/jemalloc/internal/private_symbols.txt

@@ -0,0 +1,412 @@
+a0calloc
+a0free
+a0malloc
+arena_alloc_junk_small
+arena_bin_index
+arena_bin_info
+arena_boot
+arena_dalloc
+arena_dalloc_bin
+arena_dalloc_bin_locked
+arena_dalloc_junk_large
+arena_dalloc_junk_small
+arena_dalloc_large
+arena_dalloc_large_locked
+arena_dalloc_small
+arena_dss_prec_get
+arena_dss_prec_set
+arena_malloc
+arena_malloc_large
+arena_malloc_small
+arena_mapbits_allocated_get
+arena_mapbits_binind_get
+arena_mapbits_dirty_get
+arena_mapbits_get
+arena_mapbits_large_binind_set
+arena_mapbits_large_get
+arena_mapbits_large_set
+arena_mapbits_large_size_get
+arena_mapbits_small_runind_get
+arena_mapbits_small_set
+arena_mapbits_unallocated_set
+arena_mapbits_unallocated_size_get
+arena_mapbits_unallocated_size_set
+arena_mapbits_unzeroed_get
+arena_mapbits_unzeroed_set
+arena_mapbitsp_get
+arena_mapbitsp_read
+arena_mapbitsp_write
+arena_mapp_get
+arena_maxclass
+arena_new
+arena_palloc
+arena_postfork_child
+arena_postfork_parent
+arena_prefork
+arena_prof_accum
+arena_prof_accum_impl
+arena_prof_accum_locked
+arena_prof_ctx_get
+arena_prof_ctx_set
+arena_prof_promoted
+arena_ptr_small_binind_get
+arena_purge_all
+arena_quarantine_junk_small
+arena_ralloc
+arena_ralloc_junk_large
+arena_ralloc_no_move
+arena_redzone_corruption
+arena_run_regind
+arena_salloc
+arena_stats_merge
+arena_tcache_fill_small
+arenas
+arenas_booted
+arenas_cleanup
+arenas_extend
+arenas_initialized
+arenas_lock
+arenas_tls
+arenas_tsd
+arenas_tsd_boot
+arenas_tsd_cleanup_wrapper
+arenas_tsd_get
+arenas_tsd_get_wrapper
+arenas_tsd_init_head
+arenas_tsd_set
+atomic_add_u
+atomic_add_uint32
+atomic_add_uint64
+atomic_add_z
+atomic_sub_u
+atomic_sub_uint32
+atomic_sub_uint64
+atomic_sub_z
+base_alloc
+base_boot
+base_calloc
+base_node_alloc
+base_node_dealloc
+base_postfork_child
+base_postfork_parent
+base_prefork
+bitmap_full
+bitmap_get
+bitmap_info_init
+bitmap_info_ngroups
+bitmap_init
+bitmap_set
+bitmap_sfu
+bitmap_size
+bitmap_unset
+bt_init
+buferror
+choose_arena
+choose_arena_hard
+chunk_alloc
+chunk_alloc_dss
+chunk_alloc_mmap
+chunk_boot
+chunk_dealloc
+chunk_dealloc_mmap
+chunk_dss_boot
+chunk_dss_postfork_child
+chunk_dss_postfork_parent
+chunk_dss_prec_get
+chunk_dss_prec_set
+chunk_dss_prefork
+chunk_in_dss
+chunk_npages
+chunk_postfork_child
+chunk_postfork_parent
+chunk_prefork
+chunk_unmap
+chunks_mtx
+chunks_rtree
+chunksize
+chunksize_mask
+ckh_bucket_search
+ckh_count
+ckh_delete
+ckh_evict_reloc_insert
+ckh_insert
+ckh_isearch
+ckh_iter
+ckh_new
+ckh_pointer_hash
+ckh_pointer_keycomp
+ckh_rebuild
+ckh_remove
+ckh_search
+ckh_string_hash
+ckh_string_keycomp
+ckh_try_bucket_insert
+ckh_try_insert
+ctl_boot
+ctl_bymib
+ctl_byname
+ctl_nametomib
+ctl_postfork_child
+ctl_postfork_parent
+ctl_prefork
+dss_prec_names
+extent_tree_ad_first
+extent_tree_ad_insert
+extent_tree_ad_iter
+extent_tree_ad_iter_recurse
+extent_tree_ad_iter_start
+extent_tree_ad_last
+extent_tree_ad_new
+extent_tree_ad_next
+extent_tree_ad_nsearch
+extent_tree_ad_prev
+extent_tree_ad_psearch
+extent_tree_ad_remove
+extent_tree_ad_reverse_iter
+extent_tree_ad_reverse_iter_recurse
+extent_tree_ad_reverse_iter_start
+extent_tree_ad_search
+extent_tree_szad_first
+extent_tree_szad_insert
+extent_tree_szad_iter
+extent_tree_szad_iter_recurse
+extent_tree_szad_iter_start
+extent_tree_szad_last
+extent_tree_szad_new
+extent_tree_szad_next
+extent_tree_szad_nsearch
+extent_tree_szad_prev
+extent_tree_szad_psearch
+extent_tree_szad_remove
+extent_tree_szad_reverse_iter
+extent_tree_szad_reverse_iter_recurse
+extent_tree_szad_reverse_iter_start
+extent_tree_szad_search
+get_errno
+hash
+hash_fmix_32
+hash_fmix_64
+hash_get_block_32
+hash_get_block_64
+hash_rotl_32
+hash_rotl_64
+hash_x64_128
+hash_x86_128
+hash_x86_32
+huge_allocated
+huge_boot
+huge_dalloc
+huge_dalloc_junk
+huge_malloc
+huge_mtx
+huge_ndalloc
+huge_nmalloc
+huge_palloc
+huge_postfork_child
+huge_postfork_parent
+huge_prefork
+huge_prof_ctx_get
+huge_prof_ctx_set
+huge_ralloc
+huge_ralloc_no_move
+huge_salloc
+iallocm
+icalloc
+icalloct
+idalloc
+idalloct
+imalloc
+imalloct
+ipalloc
+ipalloct
+iqalloc
+iqalloct
+iralloc
+iralloct
+iralloct_realign
+isalloc
+isthreaded
+ivsalloc
+ixalloc
+jemalloc_postfork_child
+jemalloc_postfork_parent
+jemalloc_prefork
+malloc_cprintf
+malloc_mutex_init
+malloc_mutex_lock
+malloc_mutex_postfork_child
+malloc_mutex_postfork_parent
+malloc_mutex_prefork
+malloc_mutex_unlock
+malloc_printf
+malloc_snprintf
+malloc_strtoumax
+malloc_tsd_boot
+malloc_tsd_cleanup_register
+malloc_tsd_dalloc
+malloc_tsd_malloc
+malloc_tsd_no_cleanup
+malloc_vcprintf
+malloc_vsnprintf
+malloc_write
+map_bias
+mb_write
+mutex_boot
+narenas_auto
+narenas_total
+narenas_total_get
+ncpus
+nhbins
+opt_abort
+opt_dss
+opt_junk
+opt_lg_chunk
+opt_lg_dirty_mult
+opt_lg_prof_interval
+opt_lg_prof_sample
+opt_lg_tcache_max
+opt_narenas
+opt_prof
+opt_prof_accum
+opt_prof_active
+opt_prof_final
+opt_prof_gdump
+opt_prof_leak
+opt_prof_prefix
+opt_quarantine
+opt_redzone
+opt_stats_print
+opt_tcache
+opt_utrace
+opt_valgrind
+opt_xmalloc
+opt_zero
+p2rz
+pages_purge
+pow2_ceil
+prof_backtrace
+prof_boot0
+prof_boot1
+prof_boot2
+prof_bt_count
+prof_ctx_get
+prof_ctx_set
+prof_dump_open
+prof_free
+prof_gdump
+prof_idump
+prof_interval
+prof_lookup
+prof_malloc
+prof_mdump
+prof_postfork_child
+prof_postfork_parent
+prof_prefork
+prof_promote
+prof_realloc
+prof_sample_accum_update
+prof_sample_threshold_update
+prof_tdata_booted
+prof_tdata_cleanup
+prof_tdata_get
+prof_tdata_init
+prof_tdata_initialized
+prof_tdata_tls
+prof_tdata_tsd
+prof_tdata_tsd_boot
+prof_tdata_tsd_cleanup_wrapper
+prof_tdata_tsd_get
+prof_tdata_tsd_get_wrapper
+prof_tdata_tsd_init_head
+prof_tdata_tsd_set
+quarantine
+quarantine_alloc_hook
+quarantine_boot
+quarantine_booted
+quarantine_cleanup
+quarantine_init
+quarantine_tls
+quarantine_tsd
+quarantine_tsd_boot
+quarantine_tsd_cleanup_wrapper
+quarantine_tsd_get
+quarantine_tsd_get_wrapper
+quarantine_tsd_init_head
+quarantine_tsd_set
+register_zone
+rtree_delete
+rtree_get
+rtree_get_locked
+rtree_new
+rtree_postfork_child
+rtree_postfork_parent
+rtree_prefork
+rtree_set
+s2u
+sa2u
+set_errno
+small_size2bin
+stats_cactive
+stats_cactive_add
+stats_cactive_get
+stats_cactive_sub
+stats_chunks
+stats_print
+tcache_alloc_easy
+tcache_alloc_large
+tcache_alloc_small
+tcache_alloc_small_hard
+tcache_arena_associate
+tcache_arena_dissociate
+tcache_bin_flush_large
+tcache_bin_flush_small
+tcache_bin_info
+tcache_boot0
+tcache_boot1
+tcache_booted
+tcache_create
+tcache_dalloc_large
+tcache_dalloc_small
+tcache_destroy
+tcache_enabled_booted
+tcache_enabled_get
+tcache_enabled_initialized
+tcache_enabled_set
+tcache_enabled_tls
+tcache_enabled_tsd
+tcache_enabled_tsd_boot
+tcache_enabled_tsd_cleanup_wrapper
+tcache_enabled_tsd_get
+tcache_enabled_tsd_get_wrapper
+tcache_enabled_tsd_init_head
+tcache_enabled_tsd_set
+tcache_event
+tcache_event_hard
+tcache_flush
+tcache_get
+tcache_initialized
+tcache_maxclass
+tcache_salloc
+tcache_stats_merge
+tcache_thread_cleanup
+tcache_tls
+tcache_tsd
+tcache_tsd_boot
+tcache_tsd_cleanup_wrapper
+tcache_tsd_get
+tcache_tsd_get_wrapper
+tcache_tsd_init_head
+tcache_tsd_set
+thread_allocated_booted
+thread_allocated_initialized
+thread_allocated_tls
+thread_allocated_tsd
+thread_allocated_tsd_boot
+thread_allocated_tsd_cleanup_wrapper
+thread_allocated_tsd_get
+thread_allocated_tsd_get_wrapper
+thread_allocated_tsd_init_head
+thread_allocated_tsd_set
+tsd_init_check_recursion
+tsd_init_finish
+u2rz

include/jemalloc/internal/private_unnamespace.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+for symbol in `cat $1` ; do
+  echo "#undef ${symbol}"
+done

include/jemalloc/internal/prng.h

@@ -4,7 +4,7 @@
 /*
  * Simple linear congruential pseudo-random number generator:
  *
- *   prn(y) = (a*x + c) % m
+ *   prng(y) = (a*x + c) % m
  *
  * where the following constants ensure maximal period:
  *
@@ -25,7 +25,7 @@
  *   uint32_t state      : Seed value.
  *   const uint32_t a, c : See above discussion.
  */
-#define prn32(r, lg_range, state, a, c) do {				\
+#define	prng32(r, lg_range, state, a, c) do {				\
 	assert(lg_range > 0);						\
 	assert(lg_range <= 32);						\
 									\
@@ -34,8 +34,8 @@
 	r >>= (32 - lg_range);						\
 } while (false)
 
-/* Same as prn32(), but 64 bits of pseudo-randomness, using uint64_t. */
-#define prn64(r, lg_range, state, a, c) do {				\
+/* Same as prng32(), but 64 bits of pseudo-randomness, using uint64_t. */
+#define	prng64(r, lg_range, state, a, c) do {				\
 	assert(lg_range > 0);						\
 	assert(lg_range <= 64);						\
 									\

include/jemalloc/internal/prof.h

@@ -1,4 +1,3 @@
-#ifdef JEMALLOC_PROF
 /******************************************************************************/
 #ifdef JEMALLOC_H_TYPES
 
@@ -10,28 +9,41 @@ typedef struct prof_tdata_s prof_tdata_t;
 
 /* Option defaults. */
 #define	PROF_PREFIX_DEFAULT		"jeprof"
-#define	LG_PROF_BT_MAX_DEFAULT		7
-#define	LG_PROF_SAMPLE_DEFAULT		0
+#define	LG_PROF_SAMPLE_DEFAULT		19
 #define	LG_PROF_INTERVAL_DEFAULT	-1
-#define	LG_PROF_TCMAX_DEFAULT		-1
 
 /*
- * Hard limit on stack backtrace depth.  Note that the version of
- * prof_backtrace() that is based on __builtin_return_address() necessarily has
- * a hard-coded number of backtrace frame handlers.
+ * Hard limit on stack backtrace depth.  The version of prof_backtrace() that
+ * is based on __builtin_return_address() necessarily has a hard-coded number
+ * of backtrace frame handlers, and should be kept in sync with this setting.
  */
-#if (defined(JEMALLOC_PROF_LIBGCC) || defined(JEMALLOC_PROF_LIBUNWIND))
-#  define LG_PROF_BT_MAX	((ZU(1) << (LG_SIZEOF_PTR+3)) - 1)
-#else
-#  define LG_PROF_BT_MAX	7 /* >= LG_PROF_BT_MAX_DEFAULT */
-#endif
-#define	PROF_BT_MAX		(1U << LG_PROF_BT_MAX)
+#define	PROF_BT_MAX			128
+
+/* Maximum number of backtraces to store in each per thread LRU cache. */
+#define	PROF_TCMAX			1024
 
 /* Initial hash table size. */
-#define	PROF_CKH_MINITEMS	64
+#define	PROF_CKH_MINITEMS		64
 
 /* Size of memory buffer to use when writing dump files. */
-#define	PROF_DUMP_BUF_SIZE	65536
+#define	PROF_DUMP_BUFSIZE		65536
+
+/* Size of stack-allocated buffer used by prof_printf(). */
+#define	PROF_PRINTF_BUFSIZE		128
+
+/*
+ * Number of mutexes shared among all ctx's.  No space is allocated for these
+ * unless profiling is enabled, so it's okay to over-provision.
+ */
+#define	PROF_NCTX_LOCKS			1024
+
+/*
+ * prof_tdata pointers close to NULL are used to encode state information that
+ * is used for cleaning up during thread shutdown.
+ */
+#define	PROF_TDATA_STATE_REINCARNATED	((prof_tdata_t *)(uintptr_t)1)
+#define	PROF_TDATA_STATE_PURGATORY	((prof_tdata_t *)(uintptr_t)2)
+#define	PROF_TDATA_STATE_MAX		PROF_TDATA_STATE_PURGATORY
 
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
@@ -109,8 +121,19 @@ struct prof_ctx_s {
 	/* Associated backtrace. */
 	prof_bt_t		*bt;
 
-	/* Protects cnt_merged and cnts_ql. */
-	malloc_mutex_t		lock;
+	/* Protects nlimbo, cnt_merged, and cnts_ql. */
+	malloc_mutex_t		*lock;
+
+	/*
+	 * Number of threads that currently cause this ctx to be in a state of
+	 * limbo due to one of:
+	 *   - Initializing per thread counters associated with this ctx.
+	 *   - Preparing to destroy this ctx.
+	 *   - Dumping a heap profile that includes this ctx.
+	 * nlimbo must be 1 (single destroyer) in order to safely destroy the
+	 * ctx.
+	 */
+	unsigned		nlimbo;
 
 	/* Temporary storage for summation during dump. */
 	prof_cnt_t		cnt_summed;
@@ -123,7 +146,11 @@ struct prof_ctx_s {
 	 * this context.
 	 */
 	ql_head(prof_thr_cnt_t)	cnts_ql;
+
+	/* Linkage for list of contexts to be dumped. */
+	ql_elm(prof_ctx_t)	dump_link;
 };
+typedef ql_head(prof_ctx_t) prof_ctx_list_t;
 
 struct prof_tdata_s {
 	/*
@@ -145,9 +172,14 @@ struct prof_tdata_s {
 	void			**vec;
 
 	/* Sampling state. */
-	uint64_t		prn_state;
+	uint64_t		prng_state;
 	uint64_t		threshold;
 	uint64_t		accum;
+
+	/* State used to avoid dumping while operating on prof internals. */
+	bool			enq;
+	bool			enq_idump;
+	bool			enq_gdump;
 };
 
 #endif /* JEMALLOC_H_STRUCTS */
@@ -162,14 +194,18 @@ extern bool	opt_prof;
  * to notice state changes.
  */
 extern bool	opt_prof_active;
-extern size_t	opt_lg_prof_bt_max;   /* Maximum backtrace depth. */
 extern size_t	opt_lg_prof_sample;   /* Mean bytes between samples. */
 extern ssize_t	opt_lg_prof_interval; /* lg(prof_interval). */
 extern bool	opt_prof_gdump;       /* High-water memory dumping. */
+extern bool	opt_prof_final;       /* Final profile dumping. */
 extern bool	opt_prof_leak;        /* Dump leak summary at exit. */
 extern bool	opt_prof_accum;       /* Report cumulative bytes. */
-extern ssize_t	opt_lg_prof_tcmax;    /* lg(max per thread bactrace cache) */
-extern char	opt_prof_prefix[PATH_MAX + 1];
+extern char	opt_prof_prefix[
+    /* Minimize memory bloat for non-prof builds. */
+#ifdef JEMALLOC_PROF
+    PATH_MAX +
+#endif
+    1];
 
 /*
  * Profile dump interval, measured in bytes allocated.  Each arena triggers a
@@ -186,42 +222,25 @@ extern uint64_t	prof_interval;
  */
 extern bool	prof_promote;
 
-/* (1U << opt_lg_prof_bt_max). */
-extern unsigned	prof_bt_max;
-
-/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */
-#ifndef NO_TLS
-extern __thread prof_tdata_t	*prof_tdata_tls
-    JEMALLOC_ATTR(tls_model("initial-exec"));
-#  define PROF_TCACHE_GET()	prof_tdata_tls
-#  define PROF_TCACHE_SET(v)	do {					\
-	prof_tdata_tls = (v);						\
-	pthread_setspecific(prof_tdata_tsd, (void *)(v));		\
-} while (0)
-#else
-#  define PROF_TCACHE_GET()						\
-	((prof_tdata_t *)pthread_getspecific(prof_tdata_tsd))
-#  define PROF_TCACHE_SET(v)	do {					\
-	pthread_setspecific(prof_tdata_tsd, (void *)(v));		\
-} while (0)
-#endif
-/*
- * Same contents as b2cnt_tls, but initialized such that the TSD destructor is
- * called when a thread exits, so that prof_tdata_tls contents can be merged,
- * unlinked, and deallocated.
- */
-extern pthread_key_t	prof_tdata_tsd;
-
 void	bt_init(prof_bt_t *bt, void **vec);
-void	prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max);
+void	prof_backtrace(prof_bt_t *bt, unsigned nignore);
 prof_thr_cnt_t	*prof_lookup(prof_bt_t *bt);
+#ifdef JEMALLOC_JET
+size_t	prof_bt_count(void);
+typedef int (prof_dump_open_t)(bool, const char *);
+extern prof_dump_open_t *prof_dump_open;
+#endif
 void	prof_idump(void);
 bool	prof_mdump(const char *filename);
 void	prof_gdump(void);
 prof_tdata_t	*prof_tdata_init(void);
+void	prof_tdata_cleanup(void *arg);
 void	prof_boot0(void);
 void	prof_boot1(void);
 bool	prof_boot2(void);
+void	prof_prefork(void);
+void	prof_postfork_parent(void);
+void	prof_postfork_child(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
@@ -233,13 +252,13 @@ bool	prof_boot2(void);
 									\
 	assert(size == s2u(size));					\
 									\
-	prof_tdata = PROF_TCACHE_GET();					\
-	if (prof_tdata == NULL) {					\
-		prof_tdata = prof_tdata_init();				\
-		if (prof_tdata == NULL) {				\
+	prof_tdata = prof_tdata_get(true);				\
+	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) {	\
+		if (prof_tdata != NULL)					\
+			ret = (prof_thr_cnt_t *)(uintptr_t)1U;		\
+		else							\
 			ret = NULL;					\
-			break;						\
-		}							\
+		break;							\
 	}								\
 									\
 	if (opt_prof_active == false) {					\
@@ -249,13 +268,13 @@ bool	prof_boot2(void);
 		/* Don't bother with sampling logic, since sampling   */\
 		/* interval is 1.                                     */\
 		bt_init(&bt, prof_tdata->vec);				\
-		prof_backtrace(&bt, nignore, prof_bt_max);		\
+		prof_backtrace(&bt, nignore);				\
 		ret = prof_lookup(&bt);					\
 	} else {							\
 		if (prof_tdata->threshold == 0) {			\
 			/* Initialize.  Seed the prng differently for */\
 			/* each thread.                               */\
-			prof_tdata->prn_state =				\
+			prof_tdata->prng_state =			\
 			    (uint64_t)(uintptr_t)&size;			\
 			prof_sample_threshold_update(prof_tdata);	\
 		}							\
@@ -272,7 +291,7 @@ bool	prof_boot2(void);
 		if (size >= prof_tdata->threshold -			\
 		    prof_tdata->accum) {				\
 			bt_init(&bt, prof_tdata->vec);			\
-			prof_backtrace(&bt, nignore, prof_bt_max);	\
+			prof_backtrace(&bt, nignore);			\
 			ret = prof_lookup(&bt);				\
 		} else							\
 			ret = (prof_thr_cnt_t *)(uintptr_t)1U;		\
@@ -280,23 +299,61 @@ bool	prof_boot2(void);
 } while (0)
 
 #ifndef JEMALLOC_ENABLE_INLINE
+malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *)
+
+prof_tdata_t	*prof_tdata_get(bool create);
 void	prof_sample_threshold_update(prof_tdata_t *prof_tdata);
 prof_ctx_t	*prof_ctx_get(const void *ptr);
-void	prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
+void	prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx);
 bool	prof_sample_accum_update(size_t size);
-void	prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt);
-void	prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt,
-    size_t old_size, prof_ctx_t *old_ctx);
+void	prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt);
+void	prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt,
+    size_t old_usize, prof_ctx_t *old_ctx);
 void	prof_free(const void *ptr, size_t size);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
+/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */
+malloc_tsd_externs(prof_tdata, prof_tdata_t *)
+malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL,
+    prof_tdata_cleanup)
+
+JEMALLOC_INLINE prof_tdata_t *
+prof_tdata_get(bool create)
+{
+	prof_tdata_t *prof_tdata;
+
+	cassert(config_prof);
+
+	prof_tdata = *prof_tdata_tsd_get();
+	if (create && prof_tdata == NULL)
+		prof_tdata = prof_tdata_init();
+
+	return (prof_tdata);
+}
+
 JEMALLOC_INLINE void
 prof_sample_threshold_update(prof_tdata_t *prof_tdata)
 {
+	/*
+	 * The body of this function is compiled out unless heap profiling is
+	 * enabled, so that it is possible to compile jemalloc with floating
+	 * point support completely disabled.  Avoiding floating point code is
+	 * important on memory-constrained systems, but it also enables a
+	 * workaround for versions of glibc that don't properly save/restore
+	 * floating point registers during dynamic lazy symbol loading (which
+	 * internally calls into whatever malloc implementation happens to be
+	 * integrated into the application).  Note that some compilers (e.g.
+	 * gcc 4.8) may use floating point registers for fast memory moves, so
+	 * jemalloc must be compiled with such optimizations disabled (e.g.
+	 * -mno-sse) in order for the workaround to be complete.
+	 */
+#ifdef JEMALLOC_PROF
 	uint64_t r;
 	double u;
 
+	cassert(config_prof);
+
 	/*
 	 * Compute sample threshold as a geometrically distributed random
 	 * variable with mean (2^opt_lg_prof_sample).
@@ -313,14 +370,15 @@ prof_sample_threshold_update(prof_tdata_t *prof_tdata)
 	 *   Luc Devroye
 	 *   Springer-Verlag, New York, 1986
 	 *   pp 500
-	 *   (http://cg.scs.carleton.ca/~luc/rnbookindex.html)
+	 *   (http://luc.devroye.org/rnbookindex.html)
 	 */
-	prn64(r, 53, prof_tdata->prn_state,
-	    (uint64_t)6364136223846793005LLU, (uint64_t)1442695040888963407LLU);
+	prng64(r, 53, prof_tdata->prng_state,
+	    UINT64_C(6364136223846793005), UINT64_C(1442695040888963407));
 	u = (double)r * (1.0/9007199254740992.0L);
 	prof_tdata->threshold = (uint64_t)(log(u) /
 	    log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample))))
 	    + (uint64_t)1U;
+#endif
 }
 
 JEMALLOC_INLINE prof_ctx_t *
@@ -329,13 +387,12 @@ prof_ctx_get(const void *ptr)
 	prof_ctx_t *ret;
 	arena_chunk_t *chunk;
 
+	cassert(config_prof);
 	assert(ptr != NULL);
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	if (chunk != ptr) {
 		/* Region. */
-		dassert(chunk->arena->magic == ARENA_MAGIC);
-
 		ret = arena_prof_ctx_get(ptr);
 	} else
 		ret = huge_prof_ctx_get(ptr);
@@ -344,18 +401,17 @@ prof_ctx_get(const void *ptr)
 }
 
 JEMALLOC_INLINE void
-prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
+prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx)
 {
 	arena_chunk_t *chunk;
 
+	cassert(config_prof);
 	assert(ptr != NULL);
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	if (chunk != ptr) {
 		/* Region. */
-		dassert(chunk->arena->magic == ARENA_MAGIC);
-
-		arena_prof_ctx_set(ptr, ctx);
+		arena_prof_ctx_set(ptr, usize, ctx);
 	} else
 		huge_prof_ctx_set(ptr, ctx);
 }
@@ -365,11 +421,13 @@ prof_sample_accum_update(size_t size)
 {
 	prof_tdata_t *prof_tdata;
 
+	cassert(config_prof);
 	/* Sampling logic is unnecessary if the interval is 1. */
 	assert(opt_lg_prof_sample != 0);
 
-	prof_tdata = PROF_TCACHE_GET();
-	assert(prof_tdata != NULL);
+	prof_tdata = prof_tdata_get(false);
+	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
+		return (true);
 
 	/* Take care to avoid integer overflow. */
 	if (size >= prof_tdata->threshold - prof_tdata->accum) {
@@ -388,19 +446,20 @@ prof_sample_accum_update(size_t size)
 }
 
 JEMALLOC_INLINE void
-prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt)
+prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt)
 {
 
+	cassert(config_prof);
 	assert(ptr != NULL);
-	assert(size == isalloc(ptr));
+	assert(usize == isalloc(ptr, true));
 
 	if (opt_lg_prof_sample != 0) {
-		if (prof_sample_accum_update(size)) {
+		if (prof_sample_accum_update(usize)) {
 			/*
 			 * Don't sample.  For malloc()-like allocation, it is
 			 * always possible to tell in advance how large an
 			 * object's usable size will be, so there should never
-			 * be a difference between the size passed to
+			 * be a difference between the usize passed to
 			 * PROF_ALLOC_PREP() and prof_malloc().
 			 */
 			assert((uintptr_t)cnt == (uintptr_t)1U);
@@ -408,17 +467,17 @@ prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt)
 	}
 
 	if ((uintptr_t)cnt > (uintptr_t)1U) {
-		prof_ctx_set(ptr, cnt->ctx);
+		prof_ctx_set(ptr, usize, cnt->ctx);
 
 		cnt->epoch++;
 		/*********/
 		mb_write();
 		/*********/
 		cnt->cnts.curobjs++;
-		cnt->cnts.curbytes += size;
+		cnt->cnts.curbytes += usize;
 		if (opt_prof_accum) {
 			cnt->cnts.accumobjs++;
-			cnt->cnts.accumbytes += size;
+			cnt->cnts.accumbytes += usize;
 		}
 		/*********/
 		mb_write();
@@ -428,27 +487,28 @@ prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt)
 		mb_write();
 		/*********/
 	} else
-		prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U);
+		prof_ctx_set(ptr, usize, (prof_ctx_t *)(uintptr_t)1U);
 }
 
 JEMALLOC_INLINE void
-prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt,
-    size_t old_size, prof_ctx_t *old_ctx)
+prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt,
+    size_t old_usize, prof_ctx_t *old_ctx)
 {
 	prof_thr_cnt_t *told_cnt;
 
+	cassert(config_prof);
 	assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U);
 
 	if (ptr != NULL) {
-		assert(size == isalloc(ptr));
+		assert(usize == isalloc(ptr, true));
 		if (opt_lg_prof_sample != 0) {
-			if (prof_sample_accum_update(size)) {
+			if (prof_sample_accum_update(usize)) {
 				/*
-				 * Don't sample.  The size passed to
+				 * Don't sample.  The usize passed to
 				 * PROF_ALLOC_PREP() was larger than what
 				 * actually got allocated, so a backtrace was
 				 * captured for this allocation, even though
-				 * its actual size was insufficient to cross
+				 * its actual usize was insufficient to cross
 				 * the sample threshold.
 				 */
 				cnt = (prof_thr_cnt_t *)(uintptr_t)1U;
@@ -463,10 +523,10 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt,
 			 * It's too late to propagate OOM for this realloc(),
 			 * so operate directly on old_cnt->ctx->cnt_merged.
 			 */
-			malloc_mutex_lock(&old_ctx->lock);
+			malloc_mutex_lock(old_ctx->lock);
 			old_ctx->cnt_merged.curobjs--;
-			old_ctx->cnt_merged.curbytes -= old_size;
-			malloc_mutex_unlock(&old_ctx->lock);
+			old_ctx->cnt_merged.curbytes -= old_usize;
+			malloc_mutex_unlock(old_ctx->lock);
 			told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U;
 		}
 	} else
@@ -475,23 +535,23 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt,
 	if ((uintptr_t)told_cnt > (uintptr_t)1U)
 		told_cnt->epoch++;
 	if ((uintptr_t)cnt > (uintptr_t)1U) {
-		prof_ctx_set(ptr, cnt->ctx);
+		prof_ctx_set(ptr, usize, cnt->ctx);
 		cnt->epoch++;
-	} else
-		prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U);
+	} else if (ptr != NULL)
+		prof_ctx_set(ptr, usize, (prof_ctx_t *)(uintptr_t)1U);
 	/*********/
 	mb_write();
 	/*********/
 	if ((uintptr_t)told_cnt > (uintptr_t)1U) {
 		told_cnt->cnts.curobjs--;
-		told_cnt->cnts.curbytes -= old_size;
+		told_cnt->cnts.curbytes -= old_usize;
 	}
 	if ((uintptr_t)cnt > (uintptr_t)1U) {
 		cnt->cnts.curobjs++;
-		cnt->cnts.curbytes += size;
+		cnt->cnts.curbytes += usize;
 		if (opt_prof_accum) {
 			cnt->cnts.accumobjs++;
-			cnt->cnts.accumbytes += size;
+			cnt->cnts.accumbytes += usize;
 		}
 	}
 	/*********/
@@ -510,9 +570,12 @@ prof_free(const void *ptr, size_t size)
 {
 	prof_ctx_t *ctx = prof_ctx_get(ptr);
 
+	cassert(config_prof);
+
 	if ((uintptr_t)ctx > (uintptr_t)1) {
-		assert(size == isalloc(ptr));
-		prof_thr_cnt_t *tcnt = prof_lookup(ctx->bt);
+		prof_thr_cnt_t *tcnt;
+		assert(size == isalloc(ptr, true));
+		tcnt = prof_lookup(ctx->bt);
 
 		if (tcnt != NULL) {
 			tcnt->epoch++;
@@ -533,10 +596,10 @@ prof_free(const void *ptr, size_t size)
 			 * OOM during free() cannot be propagated, so operate
 			 * directly on cnt->ctx->cnt_merged.
 			 */
-			malloc_mutex_lock(&ctx->lock);
+			malloc_mutex_lock(ctx->lock);
 			ctx->cnt_merged.curobjs--;
 			ctx->cnt_merged.curbytes -= size;
-			malloc_mutex_unlock(&ctx->lock);
+			malloc_mutex_unlock(ctx->lock);
 		}
 	}
 }
@@ -544,4 +607,3 @@ prof_free(const void *ptr, size_t size)
 
 #endif /* JEMALLOC_H_INLINES */
 /******************************************************************************/
-#endif /* JEMALLOC_PROF */

include/jemalloc/internal/public_namespace.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+for nm in `cat $1` ; do
+  n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'`
+  echo "#define	je_${n} JEMALLOC_N(${n})"
+done

include/jemalloc/internal/public_unnamespace.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+for nm in `cat $1` ; do
+  n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'`
+  echo "#undef je_${n}"
+done

include/jemalloc/internal/ql.h

@@ -1,61 +1,61 @@
 /*
  * List definitions.
  */
-#define ql_head(a_type)							\
+#define	ql_head(a_type)							\
 struct {								\
 	a_type *qlh_first;						\
 }
 
-#define ql_head_initializer(a_head) {NULL}
+#define	ql_head_initializer(a_head) {NULL}
 
-#define ql_elm(a_type)	qr(a_type)
+#define	ql_elm(a_type)	qr(a_type)
 
 /* List functions. */
-#define ql_new(a_head) do {						\
+#define	ql_new(a_head) do {						\
 	(a_head)->qlh_first = NULL;					\
 } while (0)
 
-#define ql_elm_new(a_elm, a_field) qr_new((a_elm), a_field)
+#define	ql_elm_new(a_elm, a_field) qr_new((a_elm), a_field)
 
-#define ql_first(a_head) ((a_head)->qlh_first)
+#define	ql_first(a_head) ((a_head)->qlh_first)
 
-#define ql_last(a_head, a_field)					\
+#define	ql_last(a_head, a_field)					\
 	((ql_first(a_head) != NULL)					\
 	    ? qr_prev(ql_first(a_head), a_field) : NULL)
 
-#define ql_next(a_head, a_elm, a_field)					\
+#define	ql_next(a_head, a_elm, a_field)					\
 	((ql_last(a_head, a_field) != (a_elm))				\
 	    ? qr_next((a_elm), a_field)	: NULL)
 
-#define ql_prev(a_head, a_elm, a_field)					\
+#define	ql_prev(a_head, a_elm, a_field)					\
 	((ql_first(a_head) != (a_elm)) ? qr_prev((a_elm), a_field)	\
 				       : NULL)
 
-#define ql_before_insert(a_head, a_qlelm, a_elm, a_field) do {		\
+#define	ql_before_insert(a_head, a_qlelm, a_elm, a_field) do {		\
 	qr_before_insert((a_qlelm), (a_elm), a_field);			\
 	if (ql_first(a_head) == (a_qlelm)) {				\
 		ql_first(a_head) = (a_elm);				\
 	}								\
 } while (0)
 
-#define ql_after_insert(a_qlelm, a_elm, a_field)			\
+#define	ql_after_insert(a_qlelm, a_elm, a_field)			\
 	qr_after_insert((a_qlelm), (a_elm), a_field)
 
-#define ql_head_insert(a_head, a_elm, a_field) do {			\
+#define	ql_head_insert(a_head, a_elm, a_field) do {			\
 	if (ql_first(a_head) != NULL) {					\
 		qr_before_insert(ql_first(a_head), (a_elm), a_field);	\
 	}								\
 	ql_first(a_head) = (a_elm);					\
 } while (0)
 
-#define ql_tail_insert(a_head, a_elm, a_field) do {			\
+#define	ql_tail_insert(a_head, a_elm, a_field) do {			\
 	if (ql_first(a_head) != NULL) {					\
 		qr_before_insert(ql_first(a_head), (a_elm), a_field);	\
 	}								\
 	ql_first(a_head) = qr_next((a_elm), a_field);			\
 } while (0)
 
-#define ql_remove(a_head, a_elm, a_field) do {				\
+#define	ql_remove(a_head, a_elm, a_field) do {				\
 	if (ql_first(a_head) == (a_elm)) {				\
 		ql_first(a_head) = qr_next(ql_first(a_head), a_field);	\
 	}								\
@@ -66,18 +66,18 @@ struct {								\
 	}								\
 } while (0)
 
-#define ql_head_remove(a_head, a_type, a_field) do {			\
+#define	ql_head_remove(a_head, a_type, a_field) do {			\
 	a_type *t = ql_first(a_head);					\
 	ql_remove((a_head), t, a_field);				\
 } while (0)
 
-#define ql_tail_remove(a_head, a_type, a_field) do {			\
+#define	ql_tail_remove(a_head, a_type, a_field) do {			\
 	a_type *t = ql_last(a_head, a_field);				\
 	ql_remove((a_head), t, a_field);				\
 } while (0)
 
-#define ql_foreach(a_var, a_head, a_field)				\
+#define	ql_foreach(a_var, a_head, a_field)				\
 	qr_foreach((a_var), ql_first(a_head), a_field)
 
-#define ql_reverse_foreach(a_var, a_head, a_field)			\
+#define	ql_reverse_foreach(a_var, a_head, a_field)			\
 	qr_reverse_foreach((a_var), ql_first(a_head), a_field)

include/jemalloc/internal/qr.h

@@ -1,28 +1,28 @@
 /* Ring definitions. */
-#define qr(a_type)							\
+#define	qr(a_type)							\
 struct {								\
 	a_type	*qre_next;						\
 	a_type	*qre_prev;						\
 }
 
 /* Ring functions. */
-#define qr_new(a_qr, a_field) do {					\
+#define	qr_new(a_qr, a_field) do {					\
 	(a_qr)->a_field.qre_next = (a_qr);				\
 	(a_qr)->a_field.qre_prev = (a_qr);				\
 } while (0)
 
-#define qr_next(a_qr, a_field) ((a_qr)->a_field.qre_next)
+#define	qr_next(a_qr, a_field) ((a_qr)->a_field.qre_next)
 
-#define qr_prev(a_qr, a_field) ((a_qr)->a_field.qre_prev)
+#define	qr_prev(a_qr, a_field) ((a_qr)->a_field.qre_prev)
 
-#define qr_before_insert(a_qrelm, a_qr, a_field) do {			\
+#define	qr_before_insert(a_qrelm, a_qr, a_field) do {			\
 	(a_qr)->a_field.qre_prev = (a_qrelm)->a_field.qre_prev;		\
 	(a_qr)->a_field.qre_next = (a_qrelm);				\
 	(a_qr)->a_field.qre_prev->a_field.qre_next = (a_qr);		\
 	(a_qrelm)->a_field.qre_prev = (a_qr);				\
 } while (0)
 
-#define qr_after_insert(a_qrelm, a_qr, a_field)				\
+#define	qr_after_insert(a_qrelm, a_qr, a_field)				\
     do									\
     {									\
 	(a_qr)->a_field.qre_next = (a_qrelm)->a_field.qre_next;		\
@@ -31,7 +31,7 @@ struct {								\
 	(a_qrelm)->a_field.qre_next = (a_qr);				\
     } while (0)
 
-#define qr_meld(a_qr_a, a_qr_b, a_field) do {				\
+#define	qr_meld(a_qr_a, a_qr_b, a_field) do {				\
 	void *t;							\
 	(a_qr_a)->a_field.qre_prev->a_field.qre_next = (a_qr_b);	\
 	(a_qr_b)->a_field.qre_prev->a_field.qre_next = (a_qr_a);	\
@@ -42,10 +42,10 @@ struct {								\
 
 /* qr_meld() and qr_split() are functionally equivalent, so there's no need to
  * have two copies of the code. */
-#define qr_split(a_qr_a, a_qr_b, a_field)				\
+#define	qr_split(a_qr_a, a_qr_b, a_field)				\
 	qr_meld((a_qr_a), (a_qr_b), a_field)
 
-#define qr_remove(a_qr, a_field) do {					\
+#define	qr_remove(a_qr, a_field) do {					\
 	(a_qr)->a_field.qre_prev->a_field.qre_next			\
 	    = (a_qr)->a_field.qre_next;					\
 	(a_qr)->a_field.qre_next->a_field.qre_prev			\
@@ -54,13 +54,13 @@ struct {								\
 	(a_qr)->a_field.qre_prev = (a_qr);				\
 } while (0)
 
-#define qr_foreach(var, a_qr, a_field)					\
+#define	qr_foreach(var, a_qr, a_field)					\
 	for ((var) = (a_qr);						\
 	    (var) != NULL;						\
 	    (var) = (((var)->a_field.qre_next != (a_qr))		\
 	    ? (var)->a_field.qre_next : NULL))
 
-#define qr_reverse_foreach(var, a_qr, a_field)				\
+#define	qr_reverse_foreach(var, a_qr, a_field)				\
 	for ((var) = ((a_qr) != NULL) ? qr_prev(a_qr, a_field) : NULL;	\
 	    (var) != NULL;						\
 	    (var) = (((var) != (a_qr))					\

include/jemalloc/internal/quarantine.h

@@ -0,0 +1,67 @@
+/******************************************************************************/
+#ifdef JEMALLOC_H_TYPES
+
+typedef struct quarantine_obj_s quarantine_obj_t;
+typedef struct quarantine_s quarantine_t;
+
+/* Default per thread quarantine size if valgrind is enabled. */
+#define	JEMALLOC_VALGRIND_QUARANTINE_DEFAULT	(ZU(1) << 24)
+
+#endif /* JEMALLOC_H_TYPES */
+/******************************************************************************/
+#ifdef JEMALLOC_H_STRUCTS
+
+struct quarantine_obj_s {
+	void	*ptr;
+	size_t	usize;
+};
+
+struct quarantine_s {
+	size_t			curbytes;
+	size_t			curobjs;
+	size_t			first;
+#define	LG_MAXOBJS_INIT 10
+	size_t			lg_maxobjs;
+	quarantine_obj_t	objs[1]; /* Dynamically sized ring buffer. */
+};
+
+#endif /* JEMALLOC_H_STRUCTS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_EXTERNS
+
+quarantine_t	*quarantine_init(size_t lg_maxobjs);
+void	quarantine(void *ptr);
+void	quarantine_cleanup(void *arg);
+bool	quarantine_boot(void);
+
+#endif /* JEMALLOC_H_EXTERNS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_INLINES
+
+#ifndef JEMALLOC_ENABLE_INLINE
+malloc_tsd_protos(JEMALLOC_ATTR(unused), quarantine, quarantine_t *)
+
+void	quarantine_alloc_hook(void);
+#endif
+
+#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_QUARANTINE_C_))
+malloc_tsd_externs(quarantine, quarantine_t *)
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, quarantine, quarantine_t *, NULL,
+    quarantine_cleanup)
+
+JEMALLOC_ALWAYS_INLINE void
+quarantine_alloc_hook(void)
+{
+	quarantine_t *quarantine;
+
+	assert(config_fill && opt_quarantine);
+
+	quarantine = *quarantine_tsd_get();
+	if (quarantine == NULL)
+		quarantine_init(LG_MAXOBJS_INIT);
+}
+#endif
+
+#endif /* JEMALLOC_H_INLINES */
+/******************************************************************************/
+

include/jemalloc/internal/rb.h

@@ -22,10 +22,6 @@
 #ifndef RB_H_
 #define	RB_H_
 
-#if 0
-__FBSDID("$FreeBSD: head/lib/libc/stdlib/rb.h 204493 2010-02-28 22:57:13Z jasone $");
-#endif
-
 #ifdef RB_COMPACT
 /* Node structure. */
 #define	rb_node(a_type)							\
@@ -223,88 +219,88 @@ a_prefix##reverse_iter(a_rbt_type *rbtree, a_type *start,		\
  * The following API is generated:
  *
  *   static void
- *   ex_new(ex_t *extree);
+ *   ex_new(ex_t *tree);
  *       Description: Initialize a red-black tree structure.
  *       Args:
- *         extree: Pointer to an uninitialized red-black tree object.
+ *         tree: Pointer to an uninitialized red-black tree object.
  *
  *   static ex_node_t *
- *   ex_first(ex_t *extree);
+ *   ex_first(ex_t *tree);
  *   static ex_node_t *
- *   ex_last(ex_t *extree);
- *       Description: Get the first/last node in extree.
+ *   ex_last(ex_t *tree);
+ *       Description: Get the first/last node in tree.
  *       Args:
- *         extree: Pointer to an initialized red-black tree object.
- *       Ret: First/last node in extree, or NULL if extree is empty.
+ *         tree: Pointer to an initialized red-black tree object.
+ *       Ret: First/last node in tree, or NULL if tree is empty.
  *
  *   static ex_node_t *
- *   ex_next(ex_t *extree, ex_node_t *node);
+ *   ex_next(ex_t *tree, ex_node_t *node);
  *   static ex_node_t *
- *   ex_prev(ex_t *extree, ex_node_t *node);
+ *   ex_prev(ex_t *tree, ex_node_t *node);
  *       Description: Get node's successor/predecessor.
  *       Args:
- *         extree: Pointer to an initialized red-black tree object.
- *         node : A node in extree.
- *       Ret: node's successor/predecessor in extree, or NULL if node is
+ *         tree: Pointer to an initialized red-black tree object.
+ *         node: A node in tree.
+ *       Ret: node's successor/predecessor in tree, or NULL if node is
  *            last/first.
  *
  *   static ex_node_t *
- *   ex_search(ex_t *extree, ex_node_t *key);
+ *   ex_search(ex_t *tree, ex_node_t *key);
  *       Description: Search for node that matches key.
  *       Args:
- *         extree: Pointer to an initialized red-black tree object.
- *         key  : Search key.
- *       Ret: Node in extree that matches key, or NULL if no match.
+ *         tree: Pointer to an initialized red-black tree object.
+ *         key : Search key.
+ *       Ret: Node in tree that matches key, or NULL if no match.
  *
  *   static ex_node_t *
- *   ex_nsearch(ex_t *extree, ex_node_t *key);
+ *   ex_nsearch(ex_t *tree, ex_node_t *key);
  *   static ex_node_t *
- *   ex_psearch(ex_t *extree, ex_node_t *key);
+ *   ex_psearch(ex_t *tree, ex_node_t *key);
  *       Description: Search for node that matches key.  If no match is found,
  *                    return what would be key's successor/predecessor, were
- *                    key in extree.
+ *                    key in tree.
  *       Args:
- *         extree: Pointer to an initialized red-black tree object.
- *         key   : Search key.
- *       Ret: Node in extree that matches key, or if no match, hypothetical
- *            node's successor/predecessor (NULL if no successor/predecessor).
+ *         tree: Pointer to an initialized red-black tree object.
+ *         key : Search key.
+ *       Ret: Node in tree that matches key, or if no match, hypothetical node's
+ *            successor/predecessor (NULL if no successor/predecessor).
  *
  *   static void
- *   ex_insert(ex_t *extree, ex_node_t *node);
- *       Description: Insert node into extree.
+ *   ex_insert(ex_t *tree, ex_node_t *node);
+ *       Description: Insert node into tree.
  *       Args:
- *         extree: Pointer to an initialized red-black tree object.
- *         node  : Node to be inserted into extree.
+ *         tree: Pointer to an initialized red-black tree object.
+ *         node: Node to be inserted into tree.
  *
  *   static void
- *   ex_remove(ex_t *extree, ex_node_t *node);
- *       Description: Remove node from extree.
+ *   ex_remove(ex_t *tree, ex_node_t *node);
+ *       Description: Remove node from tree.
  *       Args:
- *         extree: Pointer to an initialized red-black tree object.
- *         node  : Node in extree to be removed.
+ *         tree: Pointer to an initialized red-black tree object.
+ *         node: Node in tree to be removed.
  *
  *   static ex_node_t *
- *   ex_iter(ex_t *extree, ex_node_t *start, ex_node_t *(*cb)(ex_t *,
+ *   ex_iter(ex_t *tree, ex_node_t *start, ex_node_t *(*cb)(ex_t *,
  *     ex_node_t *, void *), void *arg);
  *   static ex_node_t *
- *   ex_reverse_iter(ex_t *extree, ex_node_t *start, ex_node *(*cb)(ex_t *,
+ *   ex_reverse_iter(ex_t *tree, ex_node_t *start, ex_node *(*cb)(ex_t *,
  *     ex_node_t *, void *), void *arg);
- *       Description: Iterate forward/backward over extree, starting at node.
- *                    If extree is modified, iteration must be immediately
+ *       Description: Iterate forward/backward over tree, starting at node.  If
+ *                    tree is modified, iteration must be immediately
  *                    terminated by the callback function that causes the
  *                    modification.
  *       Args:
- *         extree: Pointer to an initialized red-black tree object.
- *         start : Node at which to start iteration, or NULL to start at
- *                 first/last node.
- *         cb    : Callback function, which is called for each node during
- *                 iteration.  Under normal circumstances the callback function
- *                 should return NULL, which causes iteration to continue.  If a
- *                 callback function returns non-NULL, iteration is immediately
- *                 terminated and the non-NULL return value is returned by the
- *                 iterator.  This is useful for re-starting iteration after
- *                 modifying extree.
- *         arg   : Opaque pointer passed to cb().
+ *         tree : Pointer to an initialized red-black tree object.
+ *         start: Node at which to start iteration, or NULL to start at
+ *                first/last node.
+ *         cb   : Callback function, which is called for each node during
+ *                iteration.  Under normal circumstances the callback function
+ *                should return NULL, which causes iteration to continue.  If a
+ *                callback function returns non-NULL, iteration is immediately
+ *                terminated and the non-NULL return value is returned by the
+ *                iterator.  This is useful for re-starting iteration after
+ *                modifying tree.
+ *         arg  : Opaque pointer passed to cb().
  *       Ret: NULL if iteration completed, or the non-NULL callback return value
  *            that caused termination of the iteration.
  */

include/jemalloc/internal/rtree.h

@@ -14,17 +14,18 @@ typedef struct rtree_s rtree_t;
  * Size of each radix tree node (must be a power of 2).  This impacts tree
  * depth.
  */
-#if (LG_SIZEOF_PTR == 2)
-#  define RTREE_NODESIZE (1U << 14)
-#else
-#  define RTREE_NODESIZE CACHELINE
-#endif
+#define	RTREE_NODESIZE (1U << 16)
+
+typedef void *(rtree_alloc_t)(size_t);
+typedef void (rtree_dalloc_t)(void *);
 
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
 
 struct rtree_s {
+	rtree_alloc_t	*alloc;
+	rtree_dalloc_t	*dalloc;
 	malloc_mutex_t	mutex;
 	void		**root;
 	unsigned	height;
@@ -35,27 +36,31 @@ struct rtree_s {
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-rtree_t	*rtree_new(unsigned bits);
+rtree_t	*rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc);
+void	rtree_delete(rtree_t *rtree);
+void	rtree_prefork(rtree_t *rtree);
+void	rtree_postfork_parent(rtree_t *rtree);
+void	rtree_postfork_child(rtree_t *rtree);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_INLINES
 
 #ifndef JEMALLOC_ENABLE_INLINE
-#ifndef JEMALLOC_DEBUG
-void	*rtree_get_locked(rtree_t *rtree, uintptr_t key);
+#ifdef JEMALLOC_DEBUG
+uint8_t rtree_get_locked(rtree_t *rtree, uintptr_t key);
 #endif
-void	*rtree_get(rtree_t *rtree, uintptr_t key);
-bool	rtree_set(rtree_t *rtree, uintptr_t key, void *val);
+uint8_t	rtree_get(rtree_t *rtree, uintptr_t key);
+bool	rtree_set(rtree_t *rtree, uintptr_t key, uint8_t val);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_))
 #define	RTREE_GET_GENERATE(f)						\
 /* The least significant bits of the key are ignored. */		\
-JEMALLOC_INLINE void *							\
+JEMALLOC_INLINE uint8_t							\
 f(rtree_t *rtree, uintptr_t key)					\
 {									\
-	void *ret;							\
+	uint8_t ret;							\
 	uintptr_t subkey;						\
 	unsigned i, lshift, height, bits;				\
 	void **node, **child;						\
@@ -65,12 +70,12 @@ f(rtree_t *rtree, uintptr_t key)					\
 	    i < height - 1;						\
 	    i++, lshift += bits, node = child) {			\
 		bits = rtree->level2bits[i];				\
-		subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR + \
+		subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR +	\
 		    3)) - bits);					\
 		child = (void**)node[subkey];				\
 		if (child == NULL) {					\
 			RTREE_UNLOCK(&rtree->mutex);			\
-			return (NULL);					\
+			return (0);					\
 		}							\
 	}								\
 									\
@@ -81,7 +86,10 @@ f(rtree_t *rtree, uintptr_t key)					\
 	bits = rtree->level2bits[i];					\
 	subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) -	\
 	    bits);							\
-	ret = node[subkey];						\
+	{								\
+		uint8_t *leaf = (uint8_t *)node;			\
+		ret = leaf[subkey];					\
+	}								\
 	RTREE_UNLOCK(&rtree->mutex);					\
 									\
 	RTREE_GET_VALIDATE						\
@@ -120,7 +128,7 @@ RTREE_GET_GENERATE(rtree_get)
 #undef RTREE_GET_VALIDATE
 
 JEMALLOC_INLINE bool
-rtree_set(rtree_t *rtree, uintptr_t key, void *val)
+rtree_set(rtree_t *rtree, uintptr_t key, uint8_t val)
 {
 	uintptr_t subkey;
 	unsigned i, lshift, height, bits;
@@ -135,14 +143,14 @@ rtree_set(rtree_t *rtree, uintptr_t key, void *val)
 		    bits);
 		child = (void**)node[subkey];
 		if (child == NULL) {
-			child = (void**)base_alloc(sizeof(void *) <<
-			    rtree->level2bits[i+1]);
+			size_t size = ((i + 1 < height - 1) ? sizeof(void *)
+			    : (sizeof(uint8_t))) << rtree->level2bits[i+1];
+			child = (void**)rtree->alloc(size);
 			if (child == NULL) {
 				malloc_mutex_unlock(&rtree->mutex);
 				return (true);
 			}
-			memset(child, 0, sizeof(void *) <<
-			    rtree->level2bits[i+1]);
+			memset(child, 0, size);
 			node[subkey] = child;
 		}
 	}
@@ -150,7 +158,10 @@ rtree_set(rtree_t *rtree, uintptr_t key, void *val)
 	/* node is a leaf, so it contains values rather than node pointers. */
 	bits = rtree->level2bits[i];
 	subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - bits);
-	node[subkey] = val;
+	{
+		uint8_t *leaf = (uint8_t *)node;
+		leaf[subkey] = val;
+	}
 	malloc_mutex_unlock(&rtree->mutex);
 
 	return (false);

include/jemalloc/internal/size_classes.sh

@@ -0,0 +1,122 @@
+#!/bin/sh
+
+# The following limits are chosen such that they cover all supported platforms.
+
+# Range of quanta.
+lg_qmin=3
+lg_qmax=4
+
+# The range of tiny size classes is [2^lg_tmin..2^(lg_q-1)].
+lg_tmin=3
+
+# Range of page sizes.
+lg_pmin=12
+lg_pmax=16
+
+pow2() {
+  e=$1
+  pow2_result=1
+  while [ ${e} -gt 0 ] ; do
+    pow2_result=$((${pow2_result} + ${pow2_result}))
+    e=$((${e} - 1))
+  done
+}
+
+cat <<EOF
+/* This file was automatically generated by size_classes.sh. */
+/******************************************************************************/
+#ifdef JEMALLOC_H_TYPES
+
+EOF
+
+lg_q=${lg_qmin}
+while [ ${lg_q} -le ${lg_qmax} ] ; do
+  lg_t=${lg_tmin}
+  while [ ${lg_t} -le ${lg_q} ] ; do
+    lg_p=${lg_pmin}
+    while [ ${lg_p} -le ${lg_pmax} ] ; do
+      echo "#if (LG_TINY_MIN == ${lg_t} && LG_QUANTUM == ${lg_q} && LG_PAGE == ${lg_p})"
+      echo "#define	SIZE_CLASSES_DEFINED"
+      pow2 ${lg_q}; q=${pow2_result}
+      pow2 ${lg_t}; t=${pow2_result}
+      pow2 ${lg_p}; p=${pow2_result}
+      bin=0
+      psz=0
+      sz=${t}
+      delta=$((${sz} - ${psz}))
+      echo "/*  SIZE_CLASS(bin,	delta,	sz) */"
+      echo "#define	SIZE_CLASSES							\\"
+
+      # Tiny size classes.
+      while [ ${sz} -lt ${q} ] ; do
+        echo "    SIZE_CLASS(${bin},	${delta},	${sz})					\\"
+        bin=$((${bin} + 1))
+        psz=${sz}
+        sz=$((${sz} + ${sz}))
+        delta=$((${sz} - ${psz}))
+      done
+      # Quantum-multiple size classes.  For each doubling of sz, as many as 4
+      # size classes exist.  Their spacing is the greater of:
+      # - q
+      # - sz/4, where sz is a power of 2
+      while [ ${sz} -lt ${p} ] ; do
+        if [ ${sz} -ge $((${q} * 4)) ] ; then
+          i=$((${sz} / 4))
+        else
+          i=${q}
+        fi
+        next_2pow=$((${sz} * 2))
+        while [ ${sz} -lt $next_2pow ] ; do
+          echo "    SIZE_CLASS(${bin},	${delta},	${sz})					\\"
+          bin=$((${bin} + 1))
+          psz=${sz}
+          sz=$((${sz} + ${i}))
+          delta=$((${sz} - ${psz}))
+        done
+      done
+      echo
+      echo "#define	NBINS		${bin}"
+      echo "#define	SMALL_MAXCLASS	${psz}"
+      echo "#endif"
+      echo
+      lg_p=$((${lg_p} + 1))
+    done
+    lg_t=$((${lg_t} + 1))
+  done
+  lg_q=$((${lg_q} + 1))
+done
+
+cat <<EOF
+#ifndef SIZE_CLASSES_DEFINED
+#  error "No size class definitions match configuration"
+#endif
+#undef SIZE_CLASSES_DEFINED
+/*
+ * The small_size2bin lookup table uses uint8_t to encode each bin index, so we
+ * cannot support more than 256 small size classes.  Further constrain NBINS to
+ * 255 to support prof_promote, since all small size classes, plus a "not
+ * small" size class must be stored in 8 bits of arena_chunk_map_t's bits
+ * field.
+ */
+#if (NBINS > 255)
+#  error "Too many small size classes"
+#endif
+
+#endif /* JEMALLOC_H_TYPES */
+/******************************************************************************/
+#ifdef JEMALLOC_H_STRUCTS
+
+
+#endif /* JEMALLOC_H_STRUCTS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_EXTERNS
+
+
+#endif /* JEMALLOC_H_EXTERNS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_INLINES
+
+
+#endif /* JEMALLOC_H_INLINES */
+/******************************************************************************/
+EOF

include/jemalloc/internal/stats.h

@@ -1,25 +1,16 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_TYPES
 
-#define	UMAX2S_BUFSIZE	65
-
-#ifdef JEMALLOC_STATS
 typedef struct tcache_bin_stats_s tcache_bin_stats_t;
 typedef struct malloc_bin_stats_s malloc_bin_stats_t;
 typedef struct malloc_large_stats_s malloc_large_stats_t;
 typedef struct arena_stats_s arena_stats_t;
-#endif
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
 typedef struct chunk_stats_s chunk_stats_t;
-#endif
 
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
 
-#ifdef JEMALLOC_STATS
-
-#ifdef JEMALLOC_TCACHE
 struct tcache_bin_stats_s {
 	/*
 	 * Number of allocation requests that corresponded to the size of this
@@ -27,7 +18,6 @@ struct tcache_bin_stats_s {
 	 */
 	uint64_t	nrequests;
 };
-#endif
 
 struct malloc_bin_stats_s {
 	/*
@@ -52,13 +42,11 @@ struct malloc_bin_stats_s {
 	 */
 	uint64_t	nrequests;
 
-#ifdef JEMALLOC_TCACHE
 	/* Number of tcache fills from this bin. */
 	uint64_t	nfills;
 
 	/* Number of tcache flushes to this bin. */
 	uint64_t	nflushes;
-#endif
 
 	/* Total number of runs created for this bin's size class. */
 	uint64_t	nruns;
@@ -69,9 +57,6 @@ struct malloc_bin_stats_s {
 	 */
 	uint64_t	reruns;
 
-	/* High-water mark for this bin. */
-	size_t		highruns;
-
 	/* Current number of runs in this bin. */
 	size_t		curruns;
 };
@@ -93,9 +78,6 @@ struct malloc_large_stats_s {
 	 */
 	uint64_t	nrequests;
 
-	/* High-water mark for this size class. */
-	size_t		highruns;
-
 	/* Current number of runs of this size class. */
 	size_t		curruns;
 };
@@ -127,14 +109,10 @@ struct arena_stats_s {
 	 */
 	malloc_large_stats_t	*lstats;
 };
-#endif /* JEMALLOC_STATS */
 
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
 struct chunk_stats_s {
-#  ifdef JEMALLOC_STATS
 	/* Number of chunks that were allocated. */
 	uint64_t	nchunks;
-#  endif
 
 	/* High-water mark for number of chunks allocated. */
 	size_t		highchunks;
@@ -146,7 +124,6 @@ struct chunk_stats_s {
 	 */
 	size_t		curchunks;
 };
-#endif /* JEMALLOC_STATS */
 
 #endif /* JEMALLOC_H_STRUCTS */
 /******************************************************************************/
@@ -154,24 +131,14 @@ struct chunk_stats_s {
 
 extern bool	opt_stats_print;
 
-#ifdef JEMALLOC_STATS
 extern size_t	stats_cactive;
-#endif
 
-char	*u2s(uint64_t x, unsigned base, char *s);
-#ifdef JEMALLOC_STATS
-void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque,
-    const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4));
-void	malloc_printf(const char *format, ...)
-    JEMALLOC_ATTR(format(printf, 1, 2));
-#endif
 void	stats_print(void (*write)(void *, const char *), void *cbopaque,
     const char *opts);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_INLINES
-#ifdef JEMALLOC_STATS
 
 #ifndef JEMALLOC_ENABLE_INLINE
 size_t	stats_cactive_get(void);
@@ -202,6 +169,5 @@ stats_cactive_sub(size_t size)
 }
 #endif
 
-#endif /* JEMALLOC_STATS */
 #endif /* JEMALLOC_H_INLINES */
 /******************************************************************************/

include/jemalloc/internal/tcache.h

@@ -1,4 +1,3 @@
-#ifdef JEMALLOC_TCACHE
 /******************************************************************************/
 #ifdef JEMALLOC_H_TYPES
 
@@ -6,6 +5,16 @@ typedef struct tcache_bin_info_s tcache_bin_info_t;
 typedef struct tcache_bin_s tcache_bin_t;
 typedef struct tcache_s tcache_t;
 
+/*
+ * tcache pointers close to NULL are used to encode state information that is
+ * used for two purposes: preventing thread caching on a per thread basis and
+ * cleaning up during thread shutdown.
+ */
+#define	TCACHE_STATE_DISABLED		((tcache_t *)(uintptr_t)1)
+#define	TCACHE_STATE_REINCARNATED	((tcache_t *)(uintptr_t)2)
+#define	TCACHE_STATE_PURGATORY		((tcache_t *)(uintptr_t)3)
+#define	TCACHE_STATE_MAX		TCACHE_STATE_PURGATORY
+
 /*
  * Absolute maximum number of cache slots for each small bin in the thread
  * cache.  This is an additional constraint beyond that imposed as: twice the
@@ -22,17 +31,26 @@ typedef struct tcache_s tcache_t;
 #define	LG_TCACHE_MAXCLASS_DEFAULT	15
 
 /*
- * (1U << opt_lg_tcache_gc_sweep) is the approximate number of allocation
- * events between full GC sweeps (-1: disabled).  Integer rounding may cause
- * the actual number to be slightly higher, since GC is performed
- * incrementally.
+ * TCACHE_GC_SWEEP is the approximate number of allocation events between
+ * full GC sweeps.  Integer rounding may cause the actual number to be
+ * slightly higher, since GC is performed incrementally.
  */
-#define	LG_TCACHE_GC_SWEEP_DEFAULT	13
+#define	TCACHE_GC_SWEEP			8192
+
+/* Number of tcache allocation/deallocation events between incremental GCs. */
+#define	TCACHE_GC_INCR							\
+    ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1))
 
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
 
+typedef enum {
+	tcache_enabled_false   = 0, /* Enable cast to/from bool. */
+	tcache_enabled_true    = 1,
+	tcache_enabled_default = 2
+} tcache_enabled_t;
+
 /*
  * Read-only information associated with each element of tcache_t's tbins array
  * is stored separately, mainly to reduce memory usage.
@@ -42,9 +60,7 @@ struct tcache_bin_info_s {
 };
 
 struct tcache_bin_s {
-#  ifdef JEMALLOC_STATS
 	tcache_bin_stats_t tstats;
-#  endif
 	int		low_water;	/* Min # cached since last GC. */
 	unsigned	lg_fill_div;	/* Fill (ncached_max >> lg_fill_div). */
 	unsigned	ncached;	/* # of cached objects. */
@@ -52,12 +68,8 @@ struct tcache_bin_s {
 };
 
 struct tcache_s {
-#  ifdef JEMALLOC_STATS
 	ql_elm(tcache_t) link;		/* Used for aggregating stats. */
-#  endif
-#  ifdef JEMALLOC_PROF
 	uint64_t	prof_accumbytes;/* Cleared after arena_prof_accum() */
-#  endif
 	arena_t		*arena;		/* This thread's arena. */
 	unsigned	ev_cnt;		/* Event count since incremental GC. */
 	unsigned	next_gc_bin;	/* Next bin to GC. */
@@ -76,29 +88,11 @@ struct tcache_s {
 
 extern bool	opt_tcache;
 extern ssize_t	opt_lg_tcache_max;
-extern ssize_t	opt_lg_tcache_gc_sweep;
 
 extern tcache_bin_info_t	*tcache_bin_info;
 
-/* Map of thread-specific caches. */
-#ifndef NO_TLS
-extern __thread tcache_t	*tcache_tls
-    JEMALLOC_ATTR(tls_model("initial-exec"));
-#  define TCACHE_GET()	tcache_tls
-#  define TCACHE_SET(v)	do {						\
-	tcache_tls = (tcache_t *)(v);					\
-	pthread_setspecific(tcache_tsd, (void *)(v));			\
-} while (0)
-#else
-#  define TCACHE_GET()	((tcache_t *)pthread_getspecific(tcache_tsd))
-#  define TCACHE_SET(v)	do {						\
-	pthread_setspecific(tcache_tsd, (void *)(v));			\
-} while (0)
-#endif
-extern pthread_key_t		tcache_tsd;
-
 /*
- * Number of tcache bins.  There are nbins small-object bins, plus 0 or more
+ * Number of tcache bins.  There are NBINS small-object bins, plus 0 or more
  * large-object bins.
  */
 extern size_t			nhbins;
@@ -106,134 +100,178 @@ extern size_t			nhbins;
 /* Maximum cached size class. */
 extern size_t			tcache_maxclass;
 
-/* Number of tcache allocation/deallocation events between incremental GCs. */
-extern unsigned			tcache_gc_incr;
-
-void	tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-    , tcache_t *tcache
-#endif
-    );
-void	tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-    , tcache_t *tcache
-#endif
-    );
-tcache_t *tcache_create(arena_t *arena);
+size_t	tcache_salloc(const void *ptr);
+void	tcache_event_hard(tcache_t *tcache);
 void	*tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin,
     size_t binind);
+void	tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem,
+    tcache_t *tcache);
+void	tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
+    tcache_t *tcache);
+void	tcache_arena_associate(tcache_t *tcache, arena_t *arena);
+void	tcache_arena_dissociate(tcache_t *tcache);
+tcache_t *tcache_create(arena_t *arena);
 void	tcache_destroy(tcache_t *tcache);
-#ifdef JEMALLOC_STATS
+void	tcache_thread_cleanup(void *arg);
 void	tcache_stats_merge(tcache_t *tcache, arena_t *arena);
-#endif
-bool	tcache_boot(void);
+bool	tcache_boot0(void);
+bool	tcache_boot1(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_INLINES
 
 #ifndef JEMALLOC_ENABLE_INLINE
+malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache, tcache_t *)
+malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache_enabled, tcache_enabled_t)
+
 void	tcache_event(tcache_t *tcache);
-tcache_t *tcache_get(void);
+void	tcache_flush(void);
+bool	tcache_enabled_get(void);
+tcache_t *tcache_get(bool create);
+void	tcache_enabled_set(bool enabled);
 void	*tcache_alloc_easy(tcache_bin_t *tbin);
 void	*tcache_alloc_small(tcache_t *tcache, size_t size, bool zero);
 void	*tcache_alloc_large(tcache_t *tcache, size_t size, bool zero);
-void	tcache_dalloc_small(tcache_t *tcache, void *ptr);
+void	tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind);
 void	tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
-JEMALLOC_INLINE tcache_t *
-tcache_get(void)
+/* Map of thread-specific caches. */
+malloc_tsd_externs(tcache, tcache_t *)
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache, tcache_t *, NULL,
+    tcache_thread_cleanup)
+/* Per thread flag that allows thread caches to be disabled. */
+malloc_tsd_externs(tcache_enabled, tcache_enabled_t)
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache_enabled, tcache_enabled_t,
+    tcache_enabled_default, malloc_tsd_no_cleanup)
+
+JEMALLOC_INLINE void
+tcache_flush(void)
 {
 	tcache_t *tcache;
 
-	if ((isthreaded & opt_tcache) == false)
+	cassert(config_tcache);
+
+	tcache = *tcache_tsd_get();
+	if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)
+		return;
+	tcache_destroy(tcache);
+	tcache = NULL;
+	tcache_tsd_set(&tcache);
+}
+
+JEMALLOC_INLINE bool
+tcache_enabled_get(void)
+{
+	tcache_enabled_t tcache_enabled;
+
+	cassert(config_tcache);
+
+	tcache_enabled = *tcache_enabled_tsd_get();
+	if (tcache_enabled == tcache_enabled_default) {
+		tcache_enabled = (tcache_enabled_t)opt_tcache;
+		tcache_enabled_tsd_set(&tcache_enabled);
+	}
+
+	return ((bool)tcache_enabled);
+}
+
+JEMALLOC_INLINE void
+tcache_enabled_set(bool enabled)
+{
+	tcache_enabled_t tcache_enabled;
+	tcache_t *tcache;
+
+	cassert(config_tcache);
+
+	tcache_enabled = (tcache_enabled_t)enabled;
+	tcache_enabled_tsd_set(&tcache_enabled);
+	tcache = *tcache_tsd_get();
+	if (enabled) {
+		if (tcache == TCACHE_STATE_DISABLED) {
+			tcache = NULL;
+			tcache_tsd_set(&tcache);
+		}
+	} else /* disabled */ {
+		if (tcache > TCACHE_STATE_MAX) {
+			tcache_destroy(tcache);
+			tcache = NULL;
+		}
+		if (tcache == NULL) {
+			tcache = TCACHE_STATE_DISABLED;
+			tcache_tsd_set(&tcache);
+		}
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE tcache_t *
+tcache_get(bool create)
+{
+	tcache_t *tcache;
+
+	if (config_tcache == false)
+		return (NULL);
+	if (config_lazy_lock && isthreaded == false)
 		return (NULL);
 
-	tcache = TCACHE_GET();
-	if ((uintptr_t)tcache <= (uintptr_t)2) {
+	tcache = *tcache_tsd_get();
+	if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX) {
+		if (tcache == TCACHE_STATE_DISABLED)
+			return (NULL);
 		if (tcache == NULL) {
-			tcache = tcache_create(choose_arena());
-			if (tcache == NULL)
-				return (NULL);
-		} else {
-			if (tcache == (void *)(uintptr_t)1) {
+			if (create == false) {
 				/*
-				 * Make a note that an allocator function was
-				 * called after the tcache_thread_cleanup() was
-				 * called.
+				 * Creating a tcache here would cause
+				 * allocation as a side effect of free().
+				 * Ordinarily that would be okay since
+				 * tcache_create() failure is a soft failure
+				 * that doesn't propagate.  However, if TLS
+				 * data are freed via free() as in glibc,
+				 * subtle corruption could result from setting
+				 * a TLS variable after its backing memory is
+				 * freed.
 				 */
-				TCACHE_SET((uintptr_t)2);
+				return (NULL);
 			}
+			if (tcache_enabled_get() == false) {
+				tcache_enabled_set(false); /* Memoize. */
+				return (NULL);
+			}
+			return (tcache_create(choose_arena(NULL)));
+		}
+		if (tcache == TCACHE_STATE_PURGATORY) {
+			/*
+			 * Make a note that an allocator function was called
+			 * after tcache_thread_cleanup() was called.
+			 */
+			tcache = TCACHE_STATE_REINCARNATED;
+			tcache_tsd_set(&tcache);
 			return (NULL);
 		}
+		if (tcache == TCACHE_STATE_REINCARNATED)
+			return (NULL);
+		not_reached();
 	}
 
 	return (tcache);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 tcache_event(tcache_t *tcache)
 {
 
-	if (tcache_gc_incr == 0)
+	if (TCACHE_GC_INCR == 0)
 		return;
 
 	tcache->ev_cnt++;
-	assert(tcache->ev_cnt <= tcache_gc_incr);
-	if (tcache->ev_cnt == tcache_gc_incr) {
-		size_t binind = tcache->next_gc_bin;
-		tcache_bin_t *tbin = &tcache->tbins[binind];
-		tcache_bin_info_t *tbin_info = &tcache_bin_info[binind];
-
-		if (tbin->low_water > 0) {
-			/*
-			 * Flush (ceiling) 3/4 of the objects below the low
-			 * water mark.
-			 */
-			if (binind < nbins) {
-				tcache_bin_flush_small(tbin, binind,
-				    tbin->ncached - tbin->low_water +
-				    (tbin->low_water >> 2)
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-				    , tcache
-#endif
-				    );
-			} else {
-				tcache_bin_flush_large(tbin, binind,
-				    tbin->ncached - tbin->low_water +
-				    (tbin->low_water >> 2)
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-				    , tcache
-#endif
-				    );
-			}
-			/*
-			 * Reduce fill count by 2X.  Limit lg_fill_div such that
-			 * the fill count is always at least 1.
-			 */
-			if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1))
-			    >= 1)
-				tbin->lg_fill_div++;
-		} else if (tbin->low_water < 0) {
-			/*
-			 * Increase fill count by 2X.  Make sure lg_fill_div
-			 * stays greater than 0.
-			 */
-			if (tbin->lg_fill_div > 1)
-				tbin->lg_fill_div--;
-		}
-		tbin->low_water = tbin->ncached;
-
-		tcache->next_gc_bin++;
-		if (tcache->next_gc_bin == nhbins)
-			tcache->next_gc_bin = 0;
-		tcache->ev_cnt = 0;
-	}
+	assert(tcache->ev_cnt <= TCACHE_GC_INCR);
+	if (tcache->ev_cnt == TCACHE_GC_INCR)
+		tcache_event_hard(tcache);
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 tcache_alloc_easy(tcache_bin_t *tbin)
 {
 	void *ret;
@@ -249,7 +287,7 @@ tcache_alloc_easy(tcache_bin_t *tbin)
 	return (ret);
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
 {
 	void *ret;
@@ -257,37 +295,44 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
 	tcache_bin_t *tbin;
 
 	binind = SMALL_SIZE2BIN(size);
-	assert(binind < nbins);
+	assert(binind < NBINS);
 	tbin = &tcache->tbins[binind];
+	size = arena_bin_info[binind].reg_size;
 	ret = tcache_alloc_easy(tbin);
 	if (ret == NULL) {
 		ret = tcache_alloc_small_hard(tcache, tbin, binind);
 		if (ret == NULL)
 			return (NULL);
 	}
-	assert(arena_salloc(ret) == arena_bin_info[binind].reg_size);
+	assert(tcache_salloc(ret) == arena_bin_info[binind].reg_size);
 
 	if (zero == false) {
-#ifdef JEMALLOC_FILL
-		if (opt_junk)
-			memset(ret, 0xa5, size);
-		else if (opt_zero)
-			memset(ret, 0, size);
-#endif
-	} else
+		if (config_fill) {
+			if (opt_junk) {
+				arena_alloc_junk_small(ret,
+				    &arena_bin_info[binind], false);
+			} else if (opt_zero)
+				memset(ret, 0, size);
+		}
+		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
+	} else {
+		if (config_fill && opt_junk) {
+			arena_alloc_junk_small(ret, &arena_bin_info[binind],
+			    true);
+		}
+		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 		memset(ret, 0, size);
+	}
 
-#ifdef JEMALLOC_STATS
-	tbin->tstats.nrequests++;
-#endif
-#ifdef JEMALLOC_PROF
-	tcache->prof_accumbytes += arena_bin_info[binind].reg_size;
-#endif
+	if (config_stats)
+		tbin->tstats.nrequests++;
+	if (config_prof)
+		tcache->prof_accumbytes += arena_bin_info[binind].reg_size;
 	tcache_event(tcache);
 	return (ret);
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
 {
 	void *ret;
@@ -296,7 +341,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
 
 	size = PAGE_CEILING(size);
 	assert(size <= tcache_maxclass);
-	binind = nbins + (size >> PAGE_SHIFT) - 1;
+	binind = NBINS + (size >> LG_PAGE) - 1;
 	assert(binind < nhbins);
 	tbin = &tcache->tbins[binind];
 	ret = tcache_alloc_easy(tbin);
@@ -309,74 +354,53 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
 		if (ret == NULL)
 			return (NULL);
 	} else {
-#ifdef JEMALLOC_PROF
-		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
-		size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >>
-		    PAGE_SHIFT);
-		chunk->map[pageind-map_bias].bits &= ~CHUNK_MAP_CLASS_MASK;
-#endif
+		if (config_prof && prof_promote && size == PAGE) {
+			arena_chunk_t *chunk =
+			    (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
+			size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >>
+			    LG_PAGE);
+			arena_mapbits_large_binind_set(chunk, pageind,
+			    BININD_INVALID);
+		}
 		if (zero == false) {
-#ifdef JEMALLOC_FILL
-			if (opt_junk)
-				memset(ret, 0xa5, size);
-			else if (opt_zero)
-				memset(ret, 0, size);
-#endif
-		} else
+			if (config_fill) {
+				if (opt_junk)
+					memset(ret, 0xa5, size);
+				else if (opt_zero)
+					memset(ret, 0, size);
+			}
+			VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
+		} else {
+			VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 			memset(ret, 0, size);
+		}
 
-#ifdef JEMALLOC_STATS
-		tbin->tstats.nrequests++;
-#endif
-#ifdef JEMALLOC_PROF
-		tcache->prof_accumbytes += size;
-#endif
+		if (config_stats)
+			tbin->tstats.nrequests++;
+		if (config_prof)
+			tcache->prof_accumbytes += size;
 	}
 
 	tcache_event(tcache);
 	return (ret);
 }
 
-JEMALLOC_INLINE void
-tcache_dalloc_small(tcache_t *tcache, void *ptr)
+JEMALLOC_ALWAYS_INLINE void
+tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind)
 {
-	arena_t *arena;
-	arena_chunk_t *chunk;
-	arena_run_t *run;
-	arena_bin_t *bin;
 	tcache_bin_t *tbin;
 	tcache_bin_info_t *tbin_info;
-	size_t pageind, binind;
-	arena_chunk_map_t *mapelm;
 
-	assert(arena_salloc(ptr) <= small_maxclass);
+	assert(tcache_salloc(ptr) <= SMALL_MAXCLASS);
 
-	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
-	arena = chunk->arena;
-	pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT;
-	mapelm = &chunk->map[pageind-map_bias];
-	run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
-	    (mapelm->bits >> PAGE_SHIFT)) << PAGE_SHIFT));
-	dassert(run->magic == ARENA_RUN_MAGIC);
-	bin = run->bin;
-	binind = ((uintptr_t)bin - (uintptr_t)&arena->bins) /
-	    sizeof(arena_bin_t);
-	assert(binind < nbins);
-
-#ifdef JEMALLOC_FILL
-	if (opt_junk)
-		memset(ptr, 0x5a, arena_bin_info[binind].reg_size);
-#endif
+	if (config_fill && opt_junk)
+		arena_dalloc_junk_small(ptr, &arena_bin_info[binind]);
 
 	tbin = &tcache->tbins[binind];
 	tbin_info = &tcache_bin_info[binind];
 	if (tbin->ncached == tbin_info->ncached_max) {
 		tcache_bin_flush_small(tbin, binind, (tbin_info->ncached_max >>
-		    1)
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-		    , tcache
-#endif
-		    );
+		    1), tcache);
 	}
 	assert(tbin->ncached < tbin_info->ncached_max);
 	tbin->avail[tbin->ncached] = ptr;
@@ -385,38 +409,27 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr)
 	tcache_event(tcache);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size)
 {
-	arena_t *arena;
-	arena_chunk_t *chunk;
-	size_t pageind, binind;
+	size_t binind;
 	tcache_bin_t *tbin;
 	tcache_bin_info_t *tbin_info;
 
 	assert((size & PAGE_MASK) == 0);
-	assert(arena_salloc(ptr) > small_maxclass);
-	assert(arena_salloc(ptr) <= tcache_maxclass);
+	assert(tcache_salloc(ptr) > SMALL_MAXCLASS);
+	assert(tcache_salloc(ptr) <= tcache_maxclass);
 
-	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
-	arena = chunk->arena;
-	pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT;
-	binind = nbins + (size >> PAGE_SHIFT) - 1;
+	binind = NBINS + (size >> LG_PAGE) - 1;
 
-#ifdef JEMALLOC_FILL
-	if (opt_junk)
+	if (config_fill && opt_junk)
 		memset(ptr, 0x5a, size);
-#endif
 
 	tbin = &tcache->tbins[binind];
 	tbin_info = &tcache_bin_info[binind];
 	if (tbin->ncached == tbin_info->ncached_max) {
 		tcache_bin_flush_large(tbin, binind, (tbin_info->ncached_max >>
-		    1)
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-		    , tcache
-#endif
-		    );
+		    1), tcache);
 	}
 	assert(tbin->ncached < tbin_info->ncached_max);
 	tbin->avail[tbin->ncached] = ptr;
@@ -428,4 +441,3 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size)
 
 #endif /* JEMALLOC_H_INLINES */
 /******************************************************************************/
-#endif /* JEMALLOC_TCACHE */

include/jemalloc/internal/tsd.h

@@ -0,0 +1,434 @@
+/******************************************************************************/
+#ifdef JEMALLOC_H_TYPES
+
+/* Maximum number of malloc_tsd users with cleanup functions. */
+#define	MALLOC_TSD_CLEANUPS_MAX	8
+
+typedef bool (*malloc_tsd_cleanup_t)(void);
+
+#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
+    !defined(_WIN32))
+typedef struct tsd_init_block_s tsd_init_block_t;
+typedef struct tsd_init_head_s tsd_init_head_t;
+#endif
+
+/*
+ * TLS/TSD-agnostic macro-based implementation of thread-specific data.  There
+ * are four macros that support (at least) three use cases: file-private,
+ * library-private, and library-private inlined.  Following is an example
+ * library-private tsd variable:
+ *
+ * In example.h:
+ *   typedef struct {
+ *           int x;
+ *           int y;
+ *   } example_t;
+ *   #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
+ *   malloc_tsd_protos(, example, example_t *)
+ *   malloc_tsd_externs(example, example_t *)
+ * In example.c:
+ *   malloc_tsd_data(, example, example_t *, EX_INITIALIZER)
+ *   malloc_tsd_funcs(, example, example_t *, EX_INITIALIZER,
+ *       example_tsd_cleanup)
+ *
+ * The result is a set of generated functions, e.g.:
+ *
+ *   bool example_tsd_boot(void) {...}
+ *   example_t **example_tsd_get() {...}
+ *   void example_tsd_set(example_t **val) {...}
+ *
+ * Note that all of the functions deal in terms of (a_type *) rather than
+ * (a_type)  so that it is possible to support non-pointer types (unlike
+ * pthreads TSD).  example_tsd_cleanup() is passed an (a_type *) pointer that is
+ * cast to (void *).  This means that the cleanup function needs to cast *and*
+ * dereference the function argument, e.g.:
+ *
+ *   void
+ *   example_tsd_cleanup(void *arg)
+ *   {
+ *           example_t *example = *(example_t **)arg;
+ *
+ *           [...]
+ *           if ([want the cleanup function to be called again]) {
+ *                   example_tsd_set(&example);
+ *           }
+ *   }
+ *
+ * If example_tsd_set() is called within example_tsd_cleanup(), it will be
+ * called again.  This is similar to how pthreads TSD destruction works, except
+ * that pthreads only calls the cleanup function again if the value was set to
+ * non-NULL.
+ */
+
+/* malloc_tsd_protos(). */
+#define	malloc_tsd_protos(a_attr, a_name, a_type)			\
+a_attr bool								\
+a_name##_tsd_boot(void);						\
+a_attr a_type *								\
+a_name##_tsd_get(void);							\
+a_attr void								\
+a_name##_tsd_set(a_type *val);
+
+/* malloc_tsd_externs(). */
+#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
+#define	malloc_tsd_externs(a_name, a_type)				\
+extern __thread a_type	a_name##_tls;					\
+extern __thread bool	a_name##_initialized;				\
+extern bool		a_name##_booted;
+#elif (defined(JEMALLOC_TLS))
+#define	malloc_tsd_externs(a_name, a_type)				\
+extern __thread a_type	a_name##_tls;					\
+extern pthread_key_t	a_name##_tsd;					\
+extern bool		a_name##_booted;
+#elif (defined(_WIN32))
+#define	malloc_tsd_externs(a_name, a_type)				\
+extern DWORD		a_name##_tsd;					\
+extern bool		a_name##_booted;
+#else
+#define	malloc_tsd_externs(a_name, a_type)				\
+extern pthread_key_t	a_name##_tsd;					\
+extern tsd_init_head_t	a_name##_tsd_init_head;				\
+extern bool		a_name##_booted;
+#endif
+
+/* malloc_tsd_data(). */
+#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
+#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
+a_attr __thread a_type JEMALLOC_TLS_MODEL				\
+    a_name##_tls = a_initializer;					\
+a_attr __thread bool JEMALLOC_TLS_MODEL					\
+    a_name##_initialized = false;					\
+a_attr bool		a_name##_booted = false;
+#elif (defined(JEMALLOC_TLS))
+#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
+a_attr __thread a_type JEMALLOC_TLS_MODEL				\
+    a_name##_tls = a_initializer;					\
+a_attr pthread_key_t	a_name##_tsd;					\
+a_attr bool		a_name##_booted = false;
+#elif (defined(_WIN32))
+#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
+a_attr DWORD		a_name##_tsd;					\
+a_attr bool		a_name##_booted = false;
+#else
+#define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
+a_attr pthread_key_t	a_name##_tsd;					\
+a_attr tsd_init_head_t	a_name##_tsd_init_head = {			\
+	ql_head_initializer(blocks),					\
+	MALLOC_MUTEX_INITIALIZER					\
+};									\
+a_attr bool		a_name##_booted = false;
+#endif
+
+/* malloc_tsd_funcs(). */
+#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
+#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
+    a_cleanup)								\
+/* Initialization/cleanup. */						\
+a_attr bool								\
+a_name##_tsd_cleanup_wrapper(void)					\
+{									\
+									\
+	if (a_name##_initialized) {					\
+		a_name##_initialized = false;				\
+		a_cleanup(&a_name##_tls);				\
+	}								\
+	return (a_name##_initialized);					\
+}									\
+a_attr bool								\
+a_name##_tsd_boot(void)							\
+{									\
+									\
+	if (a_cleanup != malloc_tsd_no_cleanup) {			\
+		malloc_tsd_cleanup_register(				\
+		    &a_name##_tsd_cleanup_wrapper);			\
+	}								\
+	a_name##_booted = true;						\
+	return (false);							\
+}									\
+/* Get/set. */								\
+a_attr a_type *								\
+a_name##_tsd_get(void)							\
+{									\
+									\
+	assert(a_name##_booted);					\
+	return (&a_name##_tls);						\
+}									\
+a_attr void								\
+a_name##_tsd_set(a_type *val)						\
+{									\
+									\
+	assert(a_name##_booted);					\
+	a_name##_tls = (*val);						\
+	if (a_cleanup != malloc_tsd_no_cleanup)				\
+		a_name##_initialized = true;				\
+}
+#elif (defined(JEMALLOC_TLS))
+#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
+    a_cleanup)								\
+/* Initialization/cleanup. */						\
+a_attr bool								\
+a_name##_tsd_boot(void)							\
+{									\
+									\
+	if (a_cleanup != malloc_tsd_no_cleanup) {			\
+		if (pthread_key_create(&a_name##_tsd, a_cleanup) != 0)	\
+			return (true);					\
+	}								\
+	a_name##_booted = true;						\
+	return (false);							\
+}									\
+/* Get/set. */								\
+a_attr a_type *								\
+a_name##_tsd_get(void)							\
+{									\
+									\
+	assert(a_name##_booted);					\
+	return (&a_name##_tls);						\
+}									\
+a_attr void								\
+a_name##_tsd_set(a_type *val)						\
+{									\
+									\
+	assert(a_name##_booted);					\
+	a_name##_tls = (*val);						\
+	if (a_cleanup != malloc_tsd_no_cleanup) {			\
+		if (pthread_setspecific(a_name##_tsd,			\
+		    (void *)(&a_name##_tls))) {				\
+			malloc_write("<jemalloc>: Error"		\
+			    " setting TSD for "#a_name"\n");		\
+			if (opt_abort)					\
+				abort();				\
+		}							\
+	}								\
+}
+#elif (defined(_WIN32))
+#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
+    a_cleanup)								\
+/* Data structure. */							\
+typedef struct {							\
+	bool	initialized;						\
+	a_type	val;							\
+} a_name##_tsd_wrapper_t;						\
+/* Initialization/cleanup. */						\
+a_attr bool								\
+a_name##_tsd_cleanup_wrapper(void)					\
+{									\
+	a_name##_tsd_wrapper_t *wrapper;				\
+									\
+	wrapper = (a_name##_tsd_wrapper_t *) TlsGetValue(a_name##_tsd);	\
+	if (wrapper == NULL)						\
+		return (false);						\
+	if (a_cleanup != malloc_tsd_no_cleanup &&			\
+	    wrapper->initialized) {					\
+		a_type val = wrapper->val;				\
+		a_type tsd_static_data = a_initializer;			\
+		wrapper->initialized = false;				\
+		wrapper->val = tsd_static_data;				\
+		a_cleanup(&val);					\
+		if (wrapper->initialized) {				\
+			/* Trigger another cleanup round. */		\
+			return (true);					\
+		}							\
+	}								\
+	malloc_tsd_dalloc(wrapper);					\
+	return (false);							\
+}									\
+a_attr bool								\
+a_name##_tsd_boot(void)							\
+{									\
+									\
+	a_name##_tsd = TlsAlloc();					\
+	if (a_name##_tsd == TLS_OUT_OF_INDEXES)				\
+		return (true);						\
+	if (a_cleanup != malloc_tsd_no_cleanup) {			\
+		malloc_tsd_cleanup_register(				\
+		    &a_name##_tsd_cleanup_wrapper);			\
+	}								\
+	a_name##_booted = true;						\
+	return (false);							\
+}									\
+/* Get/set. */								\
+a_attr a_name##_tsd_wrapper_t *						\
+a_name##_tsd_get_wrapper(void)						\
+{									\
+	a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)	\
+	    TlsGetValue(a_name##_tsd);					\
+									\
+	if (wrapper == NULL) {						\
+		wrapper = (a_name##_tsd_wrapper_t *)			\
+		    malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t));	\
+		if (wrapper == NULL) {					\
+			malloc_write("<jemalloc>: Error allocating"	\
+			    " TSD for "#a_name"\n");			\
+			abort();					\
+		} else {						\
+			static a_type tsd_static_data = a_initializer;	\
+			wrapper->initialized = false;			\
+			wrapper->val = tsd_static_data;			\
+		}							\
+		if (!TlsSetValue(a_name##_tsd, (void *)wrapper)) {	\
+			malloc_write("<jemalloc>: Error setting"	\
+			    " TSD for "#a_name"\n");			\
+			abort();					\
+		}							\
+	}								\
+	return (wrapper);						\
+}									\
+a_attr a_type *								\
+a_name##_tsd_get(void)							\
+{									\
+	a_name##_tsd_wrapper_t *wrapper;				\
+									\
+	assert(a_name##_booted);					\
+	wrapper = a_name##_tsd_get_wrapper();				\
+	return (&wrapper->val);						\
+}									\
+a_attr void								\
+a_name##_tsd_set(a_type *val)						\
+{									\
+	a_name##_tsd_wrapper_t *wrapper;				\
+									\
+	assert(a_name##_booted);					\
+	wrapper = a_name##_tsd_get_wrapper();				\
+	wrapper->val = *(val);						\
+	if (a_cleanup != malloc_tsd_no_cleanup)				\
+		wrapper->initialized = true;				\
+}
+#else
+#define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
+    a_cleanup)								\
+/* Data structure. */							\
+typedef struct {							\
+	bool	initialized;						\
+	a_type	val;							\
+} a_name##_tsd_wrapper_t;						\
+/* Initialization/cleanup. */						\
+a_attr void								\
+a_name##_tsd_cleanup_wrapper(void *arg)					\
+{									\
+	a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)arg;\
+									\
+	if (a_cleanup != malloc_tsd_no_cleanup &&			\
+	    wrapper->initialized) {					\
+		wrapper->initialized = false;				\
+		a_cleanup(&wrapper->val);				\
+		if (wrapper->initialized) {				\
+			/* Trigger another cleanup round. */		\
+			if (pthread_setspecific(a_name##_tsd,		\
+			    (void *)wrapper)) {				\
+				malloc_write("<jemalloc>: Error"	\
+				    " setting TSD for "#a_name"\n");	\
+				if (opt_abort)				\
+					abort();			\
+			}						\
+			return;						\
+		}							\
+	}								\
+	malloc_tsd_dalloc(wrapper);					\
+}									\
+a_attr bool								\
+a_name##_tsd_boot(void)							\
+{									\
+									\
+	if (pthread_key_create(&a_name##_tsd,				\
+	    a_name##_tsd_cleanup_wrapper) != 0)				\
+		return (true);						\
+	a_name##_booted = true;						\
+	return (false);							\
+}									\
+/* Get/set. */								\
+a_attr a_name##_tsd_wrapper_t *						\
+a_name##_tsd_get_wrapper(void)						\
+{									\
+	a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)	\
+	    pthread_getspecific(a_name##_tsd);				\
+									\
+	if (wrapper == NULL) {						\
+		tsd_init_block_t block;					\
+		wrapper = tsd_init_check_recursion(			\
+		    &a_name##_tsd_init_head, &block);			\
+		if (wrapper)						\
+		    return (wrapper);					\
+		wrapper = (a_name##_tsd_wrapper_t *)			\
+		    malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t));	\
+		block.data = wrapper;					\
+		if (wrapper == NULL) {					\
+			malloc_write("<jemalloc>: Error allocating"	\
+			    " TSD for "#a_name"\n");			\
+			abort();					\
+		} else {						\
+			static a_type tsd_static_data = a_initializer;	\
+			wrapper->initialized = false;			\
+			wrapper->val = tsd_static_data;			\
+		}							\
+		if (pthread_setspecific(a_name##_tsd,			\
+		    (void *)wrapper)) {					\
+			malloc_write("<jemalloc>: Error setting"	\
+			    " TSD for "#a_name"\n");			\
+			abort();					\
+		}							\
+		tsd_init_finish(&a_name##_tsd_init_head, &block);	\
+	}								\
+	return (wrapper);						\
+}									\
+a_attr a_type *								\
+a_name##_tsd_get(void)							\
+{									\
+	a_name##_tsd_wrapper_t *wrapper;				\
+									\
+	assert(a_name##_booted);					\
+	wrapper = a_name##_tsd_get_wrapper();				\
+	return (&wrapper->val);						\
+}									\
+a_attr void								\
+a_name##_tsd_set(a_type *val)						\
+{									\
+	a_name##_tsd_wrapper_t *wrapper;				\
+									\
+	assert(a_name##_booted);					\
+	wrapper = a_name##_tsd_get_wrapper();				\
+	wrapper->val = *(val);						\
+	if (a_cleanup != malloc_tsd_no_cleanup)				\
+		wrapper->initialized = true;				\
+}
+#endif
+
+#endif /* JEMALLOC_H_TYPES */
+/******************************************************************************/
+#ifdef JEMALLOC_H_STRUCTS
+
+#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
+    !defined(_WIN32))
+struct tsd_init_block_s {
+	ql_elm(tsd_init_block_t)	link;
+	pthread_t			thread;
+	void				*data;
+};
+struct tsd_init_head_s {
+	ql_head(tsd_init_block_t)	blocks;
+	malloc_mutex_t			lock;
+};
+#endif
+
+#endif /* JEMALLOC_H_STRUCTS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_EXTERNS
+
+void	*malloc_tsd_malloc(size_t size);
+void	malloc_tsd_dalloc(void *wrapper);
+void	malloc_tsd_no_cleanup(void *);
+void	malloc_tsd_cleanup_register(bool (*f)(void));
+void	malloc_tsd_boot(void);
+#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
+    !defined(_WIN32))
+void	*tsd_init_check_recursion(tsd_init_head_t *head,
+    tsd_init_block_t *block);
+void	tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
+#endif
+
+#endif /* JEMALLOC_H_EXTERNS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_INLINES
+
+#endif /* JEMALLOC_H_INLINES */
+/******************************************************************************/

include/jemalloc/internal/util.h

@@ -0,0 +1,162 @@
+/******************************************************************************/
+#ifdef JEMALLOC_H_TYPES
+
+/* Size of stack-allocated buffer passed to buferror(). */
+#define	BUFERROR_BUF		64
+
+/*
+ * Size of stack-allocated buffer used by malloc_{,v,vc}printf().  This must be
+ * large enough for all possible uses within jemalloc.
+ */
+#define	MALLOC_PRINTF_BUFSIZE	4096
+
+/*
+ * Wrap a cpp argument that contains commas such that it isn't broken up into
+ * multiple arguments.
+ */
+#define	JEMALLOC_ARG_CONCAT(...) __VA_ARGS__
+
+/*
+ * Silence compiler warnings due to uninitialized values.  This is used
+ * wherever the compiler fails to recognize that the variable is never used
+ * uninitialized.
+ */
+#ifdef JEMALLOC_CC_SILENCE
+#  define JEMALLOC_CC_SILENCE_INIT(v) = v
+#else
+#  define JEMALLOC_CC_SILENCE_INIT(v)
+#endif
+
+/*
+ * Define a custom assert() in order to reduce the chances of deadlock during
+ * assertion failure.
+ */
+#ifndef assert
+#define	assert(e) do {							\
+	if (config_debug && !(e)) {					\
+		malloc_printf(						\
+		    "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n",	\
+		    __FILE__, __LINE__, #e);				\
+		abort();						\
+	}								\
+} while (0)
+#endif
+
+#ifndef not_reached
+#define	not_reached() do {						\
+	if (config_debug) {						\
+		malloc_printf(						\
+		    "<jemalloc>: %s:%d: Unreachable code reached\n",	\
+		    __FILE__, __LINE__);				\
+		abort();						\
+	}								\
+} while (0)
+#endif
+
+#ifndef not_implemented
+#define	not_implemented() do {						\
+	if (config_debug) {						\
+		malloc_printf("<jemalloc>: %s:%d: Not implemented\n",	\
+		    __FILE__, __LINE__);				\
+		abort();						\
+	}								\
+} while (0)
+#endif
+
+#ifndef assert_not_implemented
+#define	assert_not_implemented(e) do {					\
+	if (config_debug && !(e))					\
+		not_implemented();					\
+} while (0)
+#endif
+
+/* Use to assert a particular configuration, e.g., cassert(config_debug). */
+#define	cassert(c) do {							\
+	if ((c) == false)						\
+		not_reached();						\
+} while (0)
+
+#endif /* JEMALLOC_H_TYPES */
+/******************************************************************************/
+#ifdef JEMALLOC_H_STRUCTS
+
+#endif /* JEMALLOC_H_STRUCTS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_EXTERNS
+
+int	buferror(int err, char *buf, size_t buflen);
+uintmax_t	malloc_strtoumax(const char *restrict nptr,
+    char **restrict endptr, int base);
+void	malloc_write(const char *s);
+
+/*
+ * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating
+ * point math.
+ */
+int	malloc_vsnprintf(char *str, size_t size, const char *format,
+    va_list ap);
+int	malloc_snprintf(char *str, size_t size, const char *format, ...)
+    JEMALLOC_ATTR(format(printf, 3, 4));
+void	malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
+    const char *format, va_list ap);
+void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque,
+    const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4));
+void	malloc_printf(const char *format, ...)
+    JEMALLOC_ATTR(format(printf, 1, 2));
+
+#endif /* JEMALLOC_H_EXTERNS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_INLINES
+
+#ifndef JEMALLOC_ENABLE_INLINE
+size_t	pow2_ceil(size_t x);
+void	set_errno(int errnum);
+int	get_errno(void);
+#endif
+
+#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_))
+/* Compute the smallest power of 2 that is >= x. */
+JEMALLOC_INLINE size_t
+pow2_ceil(size_t x)
+{
+
+	x--;
+	x |= x >> 1;
+	x |= x >> 2;
+	x |= x >> 4;
+	x |= x >> 8;
+	x |= x >> 16;
+#if (LG_SIZEOF_PTR == 3)
+	x |= x >> 32;
+#endif
+	x++;
+	return (x);
+}
+
+/* Sets error code */
+JEMALLOC_INLINE void
+set_errno(int errnum)
+{
+
+#ifdef _WIN32
+	SetLastError(errnum);
+#else
+	errno = errnum;
+#endif
+}
+
+/* Get last error code */
+JEMALLOC_INLINE int
+get_errno(void)
+{
+
+#ifdef _WIN32
+	return (GetLastError());
+#else
+	return (errno);
+#endif
+}
+#endif
+
+#endif /* JEMALLOC_H_INLINES */
+/******************************************************************************/

include/jemalloc/internal/zone.h

@@ -1,23 +0,0 @@
-#ifndef JEMALLOC_ZONE
-#  error "This source file is for zones on Darwin (OS X)."
-#endif
-/******************************************************************************/
-#ifdef JEMALLOC_H_TYPES
-
-#endif /* JEMALLOC_H_TYPES */
-/******************************************************************************/
-#ifdef JEMALLOC_H_STRUCTS
-
-#endif /* JEMALLOC_H_STRUCTS */
-/******************************************************************************/
-#ifdef JEMALLOC_H_EXTERNS
-
-malloc_zone_t *create_zone(void);
-void	szone2ozone(malloc_zone_t *zone);
-
-#endif /* JEMALLOC_H_EXTERNS */
-/******************************************************************************/
-#ifdef JEMALLOC_H_INLINES
-
-#endif /* JEMALLOC_H_INLINES */
-/******************************************************************************/

include/jemalloc/jemalloc.h.in

@@ -1,66 +0,0 @@
-#ifndef JEMALLOC_H_
-#define	JEMALLOC_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <limits.h>
-#include <strings.h>
-
-#define	JEMALLOC_VERSION "@jemalloc_version@"
-#define	JEMALLOC_VERSION_MAJOR @jemalloc_version_major@
-#define	JEMALLOC_VERSION_MINOR @jemalloc_version_minor@
-#define	JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@
-#define	JEMALLOC_VERSION_NREV @jemalloc_version_nrev@
-#define	JEMALLOC_VERSION_GID "@jemalloc_version_gid@"
-
-#include "jemalloc_defs@install_suffix@.h"
-#ifndef JEMALLOC_P
-#  define JEMALLOC_P(s) s
-#endif
-
-#define	ALLOCM_LG_ALIGN(la)	(la)
-#if LG_SIZEOF_PTR == 2
-#define	ALLOCM_ALIGN(a)	(ffs(a)-1)
-#else
-#define	ALLOCM_ALIGN(a)	((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31)
-#endif
-#define	ALLOCM_ZERO	((int)0x40)
-#define	ALLOCM_NO_MOVE	((int)0x80)
-
-#define	ALLOCM_SUCCESS		0
-#define	ALLOCM_ERR_OOM		1
-#define	ALLOCM_ERR_NOT_MOVED	2
-
-extern const char	*JEMALLOC_P(malloc_conf);
-extern void		(*JEMALLOC_P(malloc_message))(void *, const char *);
-
-void	*JEMALLOC_P(malloc)(size_t size) JEMALLOC_ATTR(malloc);
-void	*JEMALLOC_P(calloc)(size_t num, size_t size) JEMALLOC_ATTR(malloc);
-int	JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size)
-    JEMALLOC_ATTR(nonnull(1));
-void	*JEMALLOC_P(realloc)(void *ptr, size_t size);
-void	JEMALLOC_P(free)(void *ptr);
-
-size_t	JEMALLOC_P(malloc_usable_size)(const void *ptr);
-void	JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *),
-    void *cbopaque, const char *opts);
-int	JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp,
-    void *newp, size_t newlen);
-int	JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp,
-    size_t *miblenp);
-int	JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp,
-    size_t *oldlenp, void *newp, size_t newlen);
-
-int	JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags)
-    JEMALLOC_ATTR(nonnull(1));
-int	JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size,
-    size_t extra, int flags) JEMALLOC_ATTR(nonnull(1));
-int	JEMALLOC_P(sallocm)(const void *ptr, size_t *rsize, int flags)
-    JEMALLOC_ATTR(nonnull(1));
-int	JEMALLOC_P(dallocm)(void *ptr, int flags) JEMALLOC_ATTR(nonnull(1));
-
-#ifdef __cplusplus
-};
-#endif
-#endif /* JEMALLOC_H_ */

include/jemalloc/jemalloc.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+objroot=$1
+
+cat <<EOF
+#ifndef JEMALLOC_H_
+#define	JEMALLOC_H_
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+EOF
+
+for hdr in jemalloc_defs.h jemalloc_rename.h jemalloc_macros.h \
+           jemalloc_protos.h jemalloc_mangle.h ; do
+  cat "${objroot}include/jemalloc/${hdr}" \
+      | grep -v 'Generated from .* by configure\.' \
+      | sed -e 's/^#define /#define	/g' \
+      | sed -e 's/ $//g'
+  echo
+done
+
+cat <<EOF
+#ifdef __cplusplus
+};
+#endif
+#endif /* JEMALLOC_H_ */
+EOF

include/jemalloc/jemalloc_defs.h.in

@@ -1,167 +1,24 @@
-#ifndef JEMALLOC_DEFS_H_
-#define	JEMALLOC_DEFS_H_
-
-/*
- * If JEMALLOC_PREFIX is defined, it will cause all public APIs to be prefixed.
- * This makes it possible, with some care, to use multiple allocators
- * simultaneously.
- *
- * In many cases it is more convenient to manually prefix allocator function
- * calls than to let macros do it automatically, particularly when using
- * multiple allocators simultaneously.  Define JEMALLOC_MANGLE before
- * #include'ing jemalloc.h in order to cause name mangling that corresponds to
- * the API prefixing.
- */
-#undef JEMALLOC_PREFIX
-#undef JEMALLOC_CPREFIX
-#if (defined(JEMALLOC_PREFIX) && defined(JEMALLOC_MANGLE))
-#undef JEMALLOC_P
-#endif
-
-/*
- * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs.
- * For shared libraries, symbol visibility mechanisms prevent these symbols
- * from being exported, but for static libraries, naming collisions are a real
- * possibility.
- */
-#undef JEMALLOC_PRIVATE_NAMESPACE
-#undef JEMALLOC_N
-
-/*
- * Hyper-threaded CPUs may need a special instruction inside spin loops in
- * order to yield to another virtual CPU.
- */
-#undef CPU_SPINWAIT
-
-/*
- * Defined if OSAtomic*() functions are available, as provided by Darwin, and
- * documented in the atomic(3) manual page.
- */
-#undef JEMALLOC_OSATOMIC
-
-/*
- * Defined if OSSpin*() functions are available, as provided by Darwin, and
- * documented in the spinlock(3) manual page.
- */
-#undef JEMALLOC_OSSPIN
-
 /* Defined if __attribute__((...)) syntax is supported. */
 #undef JEMALLOC_HAVE_ATTR
-#ifdef JEMALLOC_HAVE_ATTR
-#  define JEMALLOC_ATTR(s) __attribute__((s))
-#else
-#  define JEMALLOC_ATTR(s)
-#endif
 
-/* JEMALLOC_CC_SILENCE enables code that silences unuseful compiler warnings. */
-#undef JEMALLOC_CC_SILENCE
+/* Support the experimental API. */
+#undef JEMALLOC_EXPERIMENTAL
 
 /*
- * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables
- * inline functions.
- */
-#undef JEMALLOC_DEBUG
-
-/* JEMALLOC_STATS enables statistics calculation. */
-#undef JEMALLOC_STATS
-
-/* JEMALLOC_PROF enables allocation profiling. */
-#undef JEMALLOC_PROF
-
-/* Use libunwind for profile backtracing if defined. */
-#undef JEMALLOC_PROF_LIBUNWIND
-
-/* Use libgcc for profile backtracing if defined. */
-#undef JEMALLOC_PROF_LIBGCC
-
-/* Use gcc intrinsics for profile backtracing if defined. */
-#undef JEMALLOC_PROF_GCC
-
-/*
- * JEMALLOC_TINY enables support for tiny objects, which are smaller than one
- * quantum.
- */
-#undef JEMALLOC_TINY
-
-/*
- * JEMALLOC_TCACHE enables a thread-specific caching layer for small objects.
- * This makes it possible to allocate/deallocate objects without any locking
- * when the cache is in the steady state.
- */
-#undef JEMALLOC_TCACHE
-
-/*
- * JEMALLOC_DSS enables use of sbrk(2) to allocate chunks from the data storage
- * segment (DSS).
- */
-#undef JEMALLOC_DSS
-
-/* JEMALLOC_SWAP enables mmap()ed swap file support. */
-#undef JEMALLOC_SWAP
-
-/* Support memory filling (junk/zero). */
-#undef JEMALLOC_FILL
-
-/* Support optional abort() on OOM. */
-#undef JEMALLOC_XMALLOC
-
-/* Support SYSV semantics. */
-#undef JEMALLOC_SYSV
-
-/* Support lazy locking (avoid locking unless a second thread is launched). */
-#undef JEMALLOC_LAZY_LOCK
-
-/* Determine page size at run time if defined. */
-#undef DYNAMIC_PAGE_SHIFT
-
-/* One page is 2^STATIC_PAGE_SHIFT bytes. */
-#undef STATIC_PAGE_SHIFT
-
-/* TLS is used to map arenas and magazine caches to threads. */
-#undef NO_TLS
-
-/*
- * JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside
- * within jemalloc-owned chunks before dereferencing them.
- */
-#undef JEMALLOC_IVSALLOC
-
-/*
- * Define overrides for non-standard allocator-related functions if they
- * are present on the system.
+ * Define overrides for non-standard allocator-related functions if they are
+ * present on the system.
  */
 #undef JEMALLOC_OVERRIDE_MEMALIGN
 #undef JEMALLOC_OVERRIDE_VALLOC
 
 /*
- * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
- */
-#undef JEMALLOC_ZONE
-#undef JEMALLOC_ZONE_VERSION
-
-/* If defined, use mremap(...MREMAP_FIXED...) for huge realloc(). */
-#undef JEMALLOC_MREMAP_FIXED
-
-/*
- * Methods for purging unused pages differ between operating systems.
+ * At least Linux omits the "const" in:
  *
- *   madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages,
- *                                 such that new pages will be demand-zeroed if
- *                                 the address region is later touched.
- *   madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being
- *                             unused, such that they will be discarded rather
- *                             than swapped out.
+ *   size_t malloc_usable_size(const void *ptr);
+ *
+ * Match the operating system's prototype.
  */
-#undef JEMALLOC_PURGE_MADVISE_DONTNEED
-#undef JEMALLOC_PURGE_MADVISE_FREE
+#undef JEMALLOC_USABLE_SIZE_CONST
 
 /* sizeof(void *) == 2^LG_SIZEOF_PTR. */
 #undef LG_SIZEOF_PTR
-
-/* sizeof(int) == 2^LG_SIZEOF_INT. */
-#undef LG_SIZEOF_INT
-
-/* sizeof(long) == 2^LG_SIZEOF_LONG. */
-#undef LG_SIZEOF_LONG
-
-#endif /* JEMALLOC_DEFS_H_ */

include/jemalloc/jemalloc_macros.h.in

@@ -0,0 +1,61 @@
+#include <limits.h>
+#include <strings.h>
+
+#define	JEMALLOC_VERSION "@jemalloc_version@"
+#define	JEMALLOC_VERSION_MAJOR @jemalloc_version_major@
+#define	JEMALLOC_VERSION_MINOR @jemalloc_version_minor@
+#define	JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@
+#define	JEMALLOC_VERSION_NREV @jemalloc_version_nrev@
+#define	JEMALLOC_VERSION_GID "@jemalloc_version_gid@"
+
+#  define MALLOCX_LG_ALIGN(la)	(la)
+#  if LG_SIZEOF_PTR == 2
+#    define MALLOCX_ALIGN(a)	(ffs(a)-1)
+#  else
+#    define MALLOCX_ALIGN(a)						\
+	 ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31)
+#  endif
+#  define MALLOCX_ZERO	((int)0x40)
+/* Bias arena index bits so that 0 encodes "MALLOCX_ARENA() unspecified". */
+#  define MALLOCX_ARENA(a)	((int)(((a)+1) << 8))
+
+#ifdef JEMALLOC_EXPERIMENTAL
+#  define ALLOCM_LG_ALIGN(la)	(la)
+#  if LG_SIZEOF_PTR == 2
+#    define ALLOCM_ALIGN(a)	(ffs(a)-1)
+#  else
+#    define ALLOCM_ALIGN(a)						\
+	 ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31)
+#  endif
+#  define ALLOCM_ZERO	((int)0x40)
+#  define ALLOCM_NO_MOVE	((int)0x80)
+/* Bias arena index bits so that 0 encodes "ALLOCM_ARENA() unspecified". */
+#  define ALLOCM_ARENA(a)	((int)(((a)+1) << 8))
+#  define ALLOCM_SUCCESS	0
+#  define ALLOCM_ERR_OOM	1
+#  define ALLOCM_ERR_NOT_MOVED	2
+#endif
+
+#ifdef JEMALLOC_HAVE_ATTR
+#  define JEMALLOC_ATTR(s) __attribute__((s))
+#  define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default"))
+#  define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s))
+#  define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s))
+#  define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline)
+#elif _MSC_VER
+#  define JEMALLOC_ATTR(s)
+#  ifdef DLLEXPORT
+#    define JEMALLOC_EXPORT __declspec(dllexport)
+#  else
+#    define JEMALLOC_EXPORT __declspec(dllimport)
+#  endif
+#  define JEMALLOC_ALIGNED(s) __declspec(align(s))
+#  define JEMALLOC_SECTION(s) __declspec(allocate(s))
+#  define JEMALLOC_NOINLINE __declspec(noinline)
+#else
+#  define JEMALLOC_ATTR(s)
+#  define JEMALLOC_EXPORT
+#  define JEMALLOC_ALIGNED(s)
+#  define JEMALLOC_SECTION(s)
+#  define JEMALLOC_NOINLINE
+#endif

include/jemalloc/jemalloc_mangle.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+public_symbols_txt=$1
+symbol_prefix=$2
+
+cat <<EOF
+/*
+ * By default application code must explicitly refer to mangled symbol names,
+ * so that it is possible to use jemalloc in conjunction with another allocator
+ * in the same application.  Define JEMALLOC_MANGLE in order to cause automatic
+ * name mangling that matches the API prefixing that happened as a result of
+ * --with-mangling and/or --with-jemalloc-prefix configuration settings.
+ */
+#ifdef JEMALLOC_MANGLE
+#  ifndef JEMALLOC_NO_DEMANGLE
+#    define JEMALLOC_NO_DEMANGLE
+#  endif
+EOF
+
+for nm in `cat ${public_symbols_txt}` ; do
+  n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'`
+  echo "#  define ${n} ${symbol_prefix}${n}"
+done
+
+cat <<EOF
+#endif
+
+/*
+ * The ${symbol_prefix}* macros can be used as stable alternative names for the
+ * public jemalloc API if JEMALLOC_NO_DEMANGLE is defined.  This is primarily
+ * meant for use in jemalloc itself, but it can be used by application code to
+ * provide isolation from the name mangling specified via --with-mangling
+ * and/or --with-jemalloc-prefix.
+ */
+#ifndef JEMALLOC_NO_DEMANGLE
+EOF
+
+for nm in `cat ${public_symbols_txt}` ; do
+  n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'`
+  echo "#  undef ${symbol_prefix}${n}"
+done
+
+cat <<EOF
+#endif
+EOF

include/jemalloc/jemalloc_protos.h.in

@@ -0,0 +1,58 @@
+/*
+ * The @je_@ prefix on the following public symbol declarations is an artifact
+ * of namespace management, and should be omitted in application code unless
+ * JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle@install_suffix@.h).
+ */
+extern JEMALLOC_EXPORT const char	*@je_@malloc_conf;
+extern JEMALLOC_EXPORT void		(*@je_@malloc_message)(void *cbopaque,
+    const char *s);
+
+JEMALLOC_EXPORT void	*@je_@malloc(size_t size) JEMALLOC_ATTR(malloc);
+JEMALLOC_EXPORT void	*@je_@calloc(size_t num, size_t size)
+    JEMALLOC_ATTR(malloc);
+JEMALLOC_EXPORT int	@je_@posix_memalign(void **memptr, size_t alignment,
+    size_t size) JEMALLOC_ATTR(nonnull(1));
+JEMALLOC_EXPORT void	*@je_@aligned_alloc(size_t alignment, size_t size)
+    JEMALLOC_ATTR(malloc);
+JEMALLOC_EXPORT void	*@je_@realloc(void *ptr, size_t size);
+JEMALLOC_EXPORT void	@je_@free(void *ptr);
+
+JEMALLOC_EXPORT void	*@je_@mallocx(size_t size, int flags);
+JEMALLOC_EXPORT void	*@je_@rallocx(void *ptr, size_t size, int flags);
+JEMALLOC_EXPORT size_t	@je_@xallocx(void *ptr, size_t size, size_t extra,
+    int flags);
+JEMALLOC_EXPORT size_t	@je_@sallocx(const void *ptr, int flags);
+JEMALLOC_EXPORT void	@je_@dallocx(void *ptr, int flags);
+JEMALLOC_EXPORT size_t	@je_@nallocx(size_t size, int flags);
+
+JEMALLOC_EXPORT int	@je_@mallctl(const char *name, void *oldp,
+    size_t *oldlenp, void *newp, size_t newlen);
+JEMALLOC_EXPORT int	@je_@mallctlnametomib(const char *name, size_t *mibp,
+    size_t *miblenp);
+JEMALLOC_EXPORT int	@je_@mallctlbymib(const size_t *mib, size_t miblen,
+    void *oldp, size_t *oldlenp, void *newp, size_t newlen);
+JEMALLOC_EXPORT void	@je_@malloc_stats_print(void (*write_cb)(void *,
+    const char *), void *@je_@cbopaque, const char *opts);
+JEMALLOC_EXPORT size_t	@je_@malloc_usable_size(
+    JEMALLOC_USABLE_SIZE_CONST void *ptr);
+
+#ifdef JEMALLOC_OVERRIDE_MEMALIGN
+JEMALLOC_EXPORT void *	@je_@memalign(size_t alignment, size_t size)
+    JEMALLOC_ATTR(malloc);
+#endif
+
+#ifdef JEMALLOC_OVERRIDE_VALLOC
+JEMALLOC_EXPORT void *	@je_@valloc(size_t size) JEMALLOC_ATTR(malloc);
+#endif
+
+#ifdef JEMALLOC_EXPERIMENTAL
+JEMALLOC_EXPORT int	@je_@allocm(void **ptr, size_t *rsize, size_t size,
+    int flags) JEMALLOC_ATTR(nonnull(1));
+JEMALLOC_EXPORT int	@je_@rallocm(void **ptr, size_t *rsize, size_t size,
+    size_t extra, int flags) JEMALLOC_ATTR(nonnull(1));
+JEMALLOC_EXPORT int	@je_@sallocm(const void *ptr, size_t *rsize, int flags)
+    JEMALLOC_ATTR(nonnull(1));
+JEMALLOC_EXPORT int	@je_@dallocm(void *ptr, int flags)
+    JEMALLOC_ATTR(nonnull(1));
+JEMALLOC_EXPORT int	@je_@nallocm(size_t *rsize, size_t size, int flags);
+#endif

include/jemalloc/jemalloc_rename.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+public_symbols_txt=$1
+
+cat <<EOF
+/*
+ * Name mangling for public symbols is controlled by --with-mangling and
+ * --with-jemalloc-prefix.  With default settings the je_ prefix is stripped by
+ * these macro definitions.
+ */
+#ifndef JEMALLOC_NO_RENAME
+EOF
+
+for nm in `cat ${public_symbols_txt}` ; do
+  n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'`
+  m=`echo ${nm} |tr ':' ' ' |awk '{print $2}'`
+  echo "#  define je_${n} ${m}"
+done
+
+cat <<EOF
+#endif
+EOF

include/msvc_compat/inttypes.h

@@ -0,0 +1,313 @@
+// ISO C9x  compliant inttypes.h for Microsoft Visual Studio
+// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 
+// 
+//  Copyright (c) 2006 Alexander Chemeris
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// 
+//   1. Redistributions of source code must retain the above copyright notice,
+//      this list of conditions and the following disclaimer.
+// 
+//   2. Redistributions in binary form must reproduce the above copyright
+//      notice, this list of conditions and the following disclaimer in the
+//      documentation and/or other materials provided with the distribution.
+// 
+//   3. The name of the author may be used to endorse or promote products
+//      derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef _MSC_VER // [
+#error "Use this header only with Microsoft Visual C++ compilers!"
+#endif // _MSC_VER ]
+
+#ifndef _MSC_INTTYPES_H_ // [
+#define _MSC_INTTYPES_H_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif
+
+#include "stdint.h"
+
+// 7.8 Format conversion of integer types
+
+typedef struct {
+   intmax_t quot;
+   intmax_t rem;
+} imaxdiv_t;
+
+// 7.8.1 Macros for format specifiers
+
+#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [   See footnote 185 at page 198
+
+#ifdef _WIN64
+#  define __PRI64_PREFIX        "l"
+#  define __PRIPTR_PREFIX       "l"
+#else
+#  define __PRI64_PREFIX        "ll"
+#  define __PRIPTR_PREFIX
+#endif
+
+// The fprintf macros for signed integers are:
+#define PRId8       "d"
+#define PRIi8       "i"
+#define PRIdLEAST8  "d"
+#define PRIiLEAST8  "i"
+#define PRIdFAST8   "d"
+#define PRIiFAST8   "i"
+
+#define PRId16       "hd"
+#define PRIi16       "hi"
+#define PRIdLEAST16  "hd"
+#define PRIiLEAST16  "hi"
+#define PRIdFAST16   "hd"
+#define PRIiFAST16   "hi"
+
+#define PRId32       "d"
+#define PRIi32       "i"
+#define PRIdLEAST32  "d"
+#define PRIiLEAST32  "i"
+#define PRIdFAST32   "d"
+#define PRIiFAST32   "i"
+
+#define PRId64       __PRI64_PREFIX "d"
+#define PRIi64       __PRI64_PREFIX "i"
+#define PRIdLEAST64  __PRI64_PREFIX "d"
+#define PRIiLEAST64  __PRI64_PREFIX "i"
+#define PRIdFAST64   __PRI64_PREFIX "d"
+#define PRIiFAST64   __PRI64_PREFIX "i"
+
+#define PRIdMAX     __PRI64_PREFIX "d"
+#define PRIiMAX     __PRI64_PREFIX "i"
+
+#define PRIdPTR     __PRIPTR_PREFIX "d"
+#define PRIiPTR     __PRIPTR_PREFIX "i"
+
+// The fprintf macros for unsigned integers are:
+#define PRIo8       "o"
+#define PRIu8       "u"
+#define PRIx8       "x"
+#define PRIX8       "X"
+#define PRIoLEAST8  "o"
+#define PRIuLEAST8  "u"
+#define PRIxLEAST8  "x"
+#define PRIXLEAST8  "X"
+#define PRIoFAST8   "o"
+#define PRIuFAST8   "u"
+#define PRIxFAST8   "x"
+#define PRIXFAST8   "X"
+
+#define PRIo16       "ho"
+#define PRIu16       "hu"
+#define PRIx16       "hx"
+#define PRIX16       "hX"
+#define PRIoLEAST16  "ho"
+#define PRIuLEAST16  "hu"
+#define PRIxLEAST16  "hx"
+#define PRIXLEAST16  "hX"
+#define PRIoFAST16   "ho"
+#define PRIuFAST16   "hu"
+#define PRIxFAST16   "hx"
+#define PRIXFAST16   "hX"
+
+#define PRIo32       "o"
+#define PRIu32       "u"
+#define PRIx32       "x"
+#define PRIX32       "X"
+#define PRIoLEAST32  "o"
+#define PRIuLEAST32  "u"
+#define PRIxLEAST32  "x"
+#define PRIXLEAST32  "X"
+#define PRIoFAST32   "o"
+#define PRIuFAST32   "u"
+#define PRIxFAST32   "x"
+#define PRIXFAST32   "X"
+
+#define PRIo64       __PRI64_PREFIX "o"
+#define PRIu64       __PRI64_PREFIX "u"
+#define PRIx64       __PRI64_PREFIX "x"
+#define PRIX64       __PRI64_PREFIX "X"
+#define PRIoLEAST64  __PRI64_PREFIX "o"
+#define PRIuLEAST64  __PRI64_PREFIX "u"
+#define PRIxLEAST64  __PRI64_PREFIX "x"
+#define PRIXLEAST64  __PRI64_PREFIX "X"
+#define PRIoFAST64   __PRI64_PREFIX "o"
+#define PRIuFAST64   __PRI64_PREFIX "u"
+#define PRIxFAST64   __PRI64_PREFIX "x"
+#define PRIXFAST64   __PRI64_PREFIX "X"
+
+#define PRIoMAX     __PRI64_PREFIX "o"
+#define PRIuMAX     __PRI64_PREFIX "u"
+#define PRIxMAX     __PRI64_PREFIX "x"
+#define PRIXMAX     __PRI64_PREFIX "X"
+
+#define PRIoPTR     __PRIPTR_PREFIX "o"
+#define PRIuPTR     __PRIPTR_PREFIX "u"
+#define PRIxPTR     __PRIPTR_PREFIX "x"
+#define PRIXPTR     __PRIPTR_PREFIX "X"
+
+// The fscanf macros for signed integers are:
+#define SCNd8       "d"
+#define SCNi8       "i"
+#define SCNdLEAST8  "d"
+#define SCNiLEAST8  "i"
+#define SCNdFAST8   "d"
+#define SCNiFAST8   "i"
+
+#define SCNd16       "hd"
+#define SCNi16       "hi"
+#define SCNdLEAST16  "hd"
+#define SCNiLEAST16  "hi"
+#define SCNdFAST16   "hd"
+#define SCNiFAST16   "hi"
+
+#define SCNd32       "ld"
+#define SCNi32       "li"
+#define SCNdLEAST32  "ld"
+#define SCNiLEAST32  "li"
+#define SCNdFAST32   "ld"
+#define SCNiFAST32   "li"
+
+#define SCNd64       "I64d"
+#define SCNi64       "I64i"
+#define SCNdLEAST64  "I64d"
+#define SCNiLEAST64  "I64i"
+#define SCNdFAST64   "I64d"
+#define SCNiFAST64   "I64i"
+
+#define SCNdMAX     "I64d"
+#define SCNiMAX     "I64i"
+
+#ifdef _WIN64 // [
+#  define SCNdPTR     "I64d"
+#  define SCNiPTR     "I64i"
+#else  // _WIN64 ][
+#  define SCNdPTR     "ld"
+#  define SCNiPTR     "li"
+#endif  // _WIN64 ]
+
+// The fscanf macros for unsigned integers are:
+#define SCNo8       "o"
+#define SCNu8       "u"
+#define SCNx8       "x"
+#define SCNX8       "X"
+#define SCNoLEAST8  "o"
+#define SCNuLEAST8  "u"
+#define SCNxLEAST8  "x"
+#define SCNXLEAST8  "X"
+#define SCNoFAST8   "o"
+#define SCNuFAST8   "u"
+#define SCNxFAST8   "x"
+#define SCNXFAST8   "X"
+
+#define SCNo16       "ho"
+#define SCNu16       "hu"
+#define SCNx16       "hx"
+#define SCNX16       "hX"
+#define SCNoLEAST16  "ho"
+#define SCNuLEAST16  "hu"
+#define SCNxLEAST16  "hx"
+#define SCNXLEAST16  "hX"
+#define SCNoFAST16   "ho"
+#define SCNuFAST16   "hu"
+#define SCNxFAST16   "hx"
+#define SCNXFAST16   "hX"
+
+#define SCNo32       "lo"
+#define SCNu32       "lu"
+#define SCNx32       "lx"
+#define SCNX32       "lX"
+#define SCNoLEAST32  "lo"
+#define SCNuLEAST32  "lu"
+#define SCNxLEAST32  "lx"
+#define SCNXLEAST32  "lX"
+#define SCNoFAST32   "lo"
+#define SCNuFAST32   "lu"
+#define SCNxFAST32   "lx"
+#define SCNXFAST32   "lX"
+
+#define SCNo64       "I64o"
+#define SCNu64       "I64u"
+#define SCNx64       "I64x"
+#define SCNX64       "I64X"
+#define SCNoLEAST64  "I64o"
+#define SCNuLEAST64  "I64u"
+#define SCNxLEAST64  "I64x"
+#define SCNXLEAST64  "I64X"
+#define SCNoFAST64   "I64o"
+#define SCNuFAST64   "I64u"
+#define SCNxFAST64   "I64x"
+#define SCNXFAST64   "I64X"
+
+#define SCNoMAX     "I64o"
+#define SCNuMAX     "I64u"
+#define SCNxMAX     "I64x"
+#define SCNXMAX     "I64X"
+
+#ifdef _WIN64 // [
+#  define SCNoPTR     "I64o"
+#  define SCNuPTR     "I64u"
+#  define SCNxPTR     "I64x"
+#  define SCNXPTR     "I64X"
+#else  // _WIN64 ][
+#  define SCNoPTR     "lo"
+#  define SCNuPTR     "lu"
+#  define SCNxPTR     "lx"
+#  define SCNXPTR     "lX"
+#endif  // _WIN64 ]
+
+#endif // __STDC_FORMAT_MACROS ]
+
+// 7.8.2 Functions for greatest-width integer types
+
+// 7.8.2.1 The imaxabs function
+#define imaxabs _abs64
+
+// 7.8.2.2 The imaxdiv function
+
+// This is modified version of div() function from Microsoft's div.c found
+// in %MSVC.NET%\crt\src\div.c
+#ifdef STATIC_IMAXDIV // [
+static
+#else // STATIC_IMAXDIV ][
+_inline
+#endif // STATIC_IMAXDIV ]
+imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom)
+{
+   imaxdiv_t result;
+
+   result.quot = numer / denom;
+   result.rem = numer % denom;
+
+   if (numer < 0 && result.rem > 0) {
+      // did division wrong; must fix up
+      ++result.quot;
+      result.rem -= denom;
+   }
+
+   return result;
+}
+
+// 7.8.2.3 The strtoimax and strtoumax functions
+#define strtoimax _strtoi64
+#define strtoumax _strtoui64
+
+// 7.8.2.4 The wcstoimax and wcstoumax functions
+#define wcstoimax _wcstoi64
+#define wcstoumax _wcstoui64
+
+
+#endif // _MSC_INTTYPES_H_ ]

include/msvc_compat/stdbool.h

@@ -0,0 +1,16 @@
+#ifndef stdbool_h
+#define stdbool_h
+
+#include <wtypes.h>
+
+/* MSVC doesn't define _Bool or bool in C, but does have BOOL */
+/* Note this doesn't pass autoconf's test because (bool) 0.5 != true */
+typedef BOOL _Bool;
+
+#define bool _Bool
+#define true 1
+#define false 0
+
+#define __bool_true_false_are_defined 1
+
+#endif /* stdbool_h */

include/msvc_compat/stdint.h

@@ -0,0 +1,247 @@
+// ISO C9x  compliant stdint.h for Microsoft Visual Studio
+// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 
+// 
+//  Copyright (c) 2006-2008 Alexander Chemeris
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// 
+//   1. Redistributions of source code must retain the above copyright notice,
+//      this list of conditions and the following disclaimer.
+// 
+//   2. Redistributions in binary form must reproduce the above copyright
+//      notice, this list of conditions and the following disclaimer in the
+//      documentation and/or other materials provided with the distribution.
+// 
+//   3. The name of the author may be used to endorse or promote products
+//      derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef _MSC_VER // [
+#error "Use this header only with Microsoft Visual C++ compilers!"
+#endif // _MSC_VER ]
+
+#ifndef _MSC_STDINT_H_ // [
+#define _MSC_STDINT_H_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif
+
+#include <limits.h>
+
+// For Visual Studio 6 in C++ mode and for many Visual Studio versions when
+// compiling for ARM we should wrap <wchar.h> include with 'extern "C++" {}'
+// or compiler give many errors like this:
+//   error C2733: second C linkage of overloaded function 'wmemchr' not allowed
+#ifdef __cplusplus
+extern "C" {
+#endif
+#  include <wchar.h>
+#ifdef __cplusplus
+}
+#endif
+
+// Define _W64 macros to mark types changing their size, like intptr_t.
+#ifndef _W64
+#  if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300
+#     define _W64 __w64
+#  else
+#     define _W64
+#  endif
+#endif
+
+
+// 7.18.1 Integer types
+
+// 7.18.1.1 Exact-width integer types
+
+// Visual Studio 6 and Embedded Visual C++ 4 doesn't
+// realize that, e.g. char has the same size as __int8
+// so we give up on __intX for them.
+#if (_MSC_VER < 1300)
+   typedef signed char       int8_t;
+   typedef signed short      int16_t;
+   typedef signed int        int32_t;
+   typedef unsigned char     uint8_t;
+   typedef unsigned short    uint16_t;
+   typedef unsigned int      uint32_t;
+#else
+   typedef signed __int8     int8_t;
+   typedef signed __int16    int16_t;
+   typedef signed __int32    int32_t;
+   typedef unsigned __int8   uint8_t;
+   typedef unsigned __int16  uint16_t;
+   typedef unsigned __int32  uint32_t;
+#endif
+typedef signed __int64       int64_t;
+typedef unsigned __int64     uint64_t;
+
+
+// 7.18.1.2 Minimum-width integer types
+typedef int8_t    int_least8_t;
+typedef int16_t   int_least16_t;
+typedef int32_t   int_least32_t;
+typedef int64_t   int_least64_t;
+typedef uint8_t   uint_least8_t;
+typedef uint16_t  uint_least16_t;
+typedef uint32_t  uint_least32_t;
+typedef uint64_t  uint_least64_t;
+
+// 7.18.1.3 Fastest minimum-width integer types
+typedef int8_t    int_fast8_t;
+typedef int16_t   int_fast16_t;
+typedef int32_t   int_fast32_t;
+typedef int64_t   int_fast64_t;
+typedef uint8_t   uint_fast8_t;
+typedef uint16_t  uint_fast16_t;
+typedef uint32_t  uint_fast32_t;
+typedef uint64_t  uint_fast64_t;
+
+// 7.18.1.4 Integer types capable of holding object pointers
+#ifdef _WIN64 // [
+   typedef signed __int64    intptr_t;
+   typedef unsigned __int64  uintptr_t;
+#else // _WIN64 ][
+   typedef _W64 signed int   intptr_t;
+   typedef _W64 unsigned int uintptr_t;
+#endif // _WIN64 ]
+
+// 7.18.1.5 Greatest-width integer types
+typedef int64_t   intmax_t;
+typedef uint64_t  uintmax_t;
+
+
+// 7.18.2 Limits of specified-width integer types
+
+#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [   See footnote 220 at page 257 and footnote 221 at page 259
+
+// 7.18.2.1 Limits of exact-width integer types
+#define INT8_MIN     ((int8_t)_I8_MIN)
+#define INT8_MAX     _I8_MAX
+#define INT16_MIN    ((int16_t)_I16_MIN)
+#define INT16_MAX    _I16_MAX
+#define INT32_MIN    ((int32_t)_I32_MIN)
+#define INT32_MAX    _I32_MAX
+#define INT64_MIN    ((int64_t)_I64_MIN)
+#define INT64_MAX    _I64_MAX
+#define UINT8_MAX    _UI8_MAX
+#define UINT16_MAX   _UI16_MAX
+#define UINT32_MAX   _UI32_MAX
+#define UINT64_MAX   _UI64_MAX
+
+// 7.18.2.2 Limits of minimum-width integer types
+#define INT_LEAST8_MIN    INT8_MIN
+#define INT_LEAST8_MAX    INT8_MAX
+#define INT_LEAST16_MIN   INT16_MIN
+#define INT_LEAST16_MAX   INT16_MAX
+#define INT_LEAST32_MIN   INT32_MIN
+#define INT_LEAST32_MAX   INT32_MAX
+#define INT_LEAST64_MIN   INT64_MIN
+#define INT_LEAST64_MAX   INT64_MAX
+#define UINT_LEAST8_MAX   UINT8_MAX
+#define UINT_LEAST16_MAX  UINT16_MAX
+#define UINT_LEAST32_MAX  UINT32_MAX
+#define UINT_LEAST64_MAX  UINT64_MAX
+
+// 7.18.2.3 Limits of fastest minimum-width integer types
+#define INT_FAST8_MIN    INT8_MIN
+#define INT_FAST8_MAX    INT8_MAX
+#define INT_FAST16_MIN   INT16_MIN
+#define INT_FAST16_MAX   INT16_MAX
+#define INT_FAST32_MIN   INT32_MIN
+#define INT_FAST32_MAX   INT32_MAX
+#define INT_FAST64_MIN   INT64_MIN
+#define INT_FAST64_MAX   INT64_MAX
+#define UINT_FAST8_MAX   UINT8_MAX
+#define UINT_FAST16_MAX  UINT16_MAX
+#define UINT_FAST32_MAX  UINT32_MAX
+#define UINT_FAST64_MAX  UINT64_MAX
+
+// 7.18.2.4 Limits of integer types capable of holding object pointers
+#ifdef _WIN64 // [
+#  define INTPTR_MIN   INT64_MIN
+#  define INTPTR_MAX   INT64_MAX
+#  define UINTPTR_MAX  UINT64_MAX
+#else // _WIN64 ][
+#  define INTPTR_MIN   INT32_MIN
+#  define INTPTR_MAX   INT32_MAX
+#  define UINTPTR_MAX  UINT32_MAX
+#endif // _WIN64 ]
+
+// 7.18.2.5 Limits of greatest-width integer types
+#define INTMAX_MIN   INT64_MIN
+#define INTMAX_MAX   INT64_MAX
+#define UINTMAX_MAX  UINT64_MAX
+
+// 7.18.3 Limits of other integer types
+
+#ifdef _WIN64 // [
+#  define PTRDIFF_MIN  _I64_MIN
+#  define PTRDIFF_MAX  _I64_MAX
+#else  // _WIN64 ][
+#  define PTRDIFF_MIN  _I32_MIN
+#  define PTRDIFF_MAX  _I32_MAX
+#endif  // _WIN64 ]
+
+#define SIG_ATOMIC_MIN  INT_MIN
+#define SIG_ATOMIC_MAX  INT_MAX
+
+#ifndef SIZE_MAX // [
+#  ifdef _WIN64 // [
+#     define SIZE_MAX  _UI64_MAX
+#  else // _WIN64 ][
+#     define SIZE_MAX  _UI32_MAX
+#  endif // _WIN64 ]
+#endif // SIZE_MAX ]
+
+// WCHAR_MIN and WCHAR_MAX are also defined in <wchar.h>
+#ifndef WCHAR_MIN // [
+#  define WCHAR_MIN  0
+#endif  // WCHAR_MIN ]
+#ifndef WCHAR_MAX // [
+#  define WCHAR_MAX  _UI16_MAX
+#endif  // WCHAR_MAX ]
+
+#define WINT_MIN  0
+#define WINT_MAX  _UI16_MAX
+
+#endif // __STDC_LIMIT_MACROS ]
+
+
+// 7.18.4 Limits of other integer types
+
+#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [   See footnote 224 at page 260
+
+// 7.18.4.1 Macros for minimum-width integer constants
+
+#define INT8_C(val)  val##i8
+#define INT16_C(val) val##i16
+#define INT32_C(val) val##i32
+#define INT64_C(val) val##i64
+
+#define UINT8_C(val)  val##ui8
+#define UINT16_C(val) val##ui16
+#define UINT32_C(val) val##ui32
+#define UINT64_C(val) val##ui64
+
+// 7.18.4.2 Macros for greatest-width integer constants
+#define INTMAX_C   INT64_C
+#define UINTMAX_C  UINT64_C
+
+#endif // __STDC_CONSTANT_MACROS ]
+
+
+#endif // _MSC_STDINT_H_ ]

include/msvc_compat/strings.h

@@ -0,0 +1,23 @@
+#ifndef strings_h
+#define strings_h
+
+/* MSVC doesn't define ffs/ffsl. This dummy strings.h header is provided
+ * for both */
+#include <intrin.h>
+#pragma intrinsic(_BitScanForward)
+static __forceinline int ffsl(long x)
+{
+	unsigned long i;
+
+	if (_BitScanForward(&i, x))
+		return (i + 1);
+	return (0);
+}
+
+static __forceinline int ffs(int x)
+{
+
+	return (ffsl(x));
+}
+
+#endif

src/base.c

@@ -4,7 +4,7 @@
 /******************************************************************************/
 /* Data. */
 
-malloc_mutex_t	base_mtx;
+static malloc_mutex_t	base_mtx;
 
 /*
  * Current pages that are being used for internal memory allocations.  These
@@ -32,7 +32,8 @@ base_pages_alloc(size_t minsize)
 	assert(minsize != 0);
 	csize = CHUNK_CEILING(minsize);
 	zero = false;
-	base_pages = chunk_alloc(csize, true, &zero);
+	base_pages = chunk_alloc(csize, chunksize, true, &zero,
+	    chunk_dss_prec_get());
 	if (base_pages == NULL)
 		return (true);
 	base_next_addr = base_pages;
@@ -62,6 +63,18 @@ base_alloc(size_t size)
 	ret = base_next_addr;
 	base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
 	malloc_mutex_unlock(&base_mtx);
+	VALGRIND_MAKE_MEM_UNDEFINED(ret, csize);
+
+	return (ret);
+}
+
+void *
+base_calloc(size_t number, size_t size)
+{
+	void *ret = base_alloc(number * size);
+
+	if (ret != NULL)
+		memset(ret, 0, number * size);
 
 	return (ret);
 }
@@ -76,6 +89,7 @@ base_node_alloc(void)
 		ret = base_nodes;
 		base_nodes = *(extent_node_t **)ret;
 		malloc_mutex_unlock(&base_mtx);
+		VALGRIND_MAKE_MEM_UNDEFINED(ret, sizeof(extent_node_t));
 	} else {
 		malloc_mutex_unlock(&base_mtx);
 		ret = (extent_node_t *)base_alloc(sizeof(extent_node_t));
@@ -88,6 +102,7 @@ void
 base_node_dealloc(extent_node_t *node)
 {
 
+	VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
 	malloc_mutex_lock(&base_mtx);
 	*(extent_node_t **)node = base_nodes;
 	base_nodes = node;
@@ -104,3 +119,24 @@ base_boot(void)
 
 	return (false);
 }
+
+void
+base_prefork(void)
+{
+
+	malloc_mutex_prefork(&base_mtx);
+}
+
+void
+base_postfork_parent(void)
+{
+
+	malloc_mutex_postfork_parent(&base_mtx);
+}
+
+void
+base_postfork_child(void)
+{
+
+	malloc_mutex_postfork_child(&base_mtx);
+}

src/bitmap.c

@@ -1,4 +1,4 @@
-#define JEMALLOC_BITMAP_C_
+#define	JEMALLOC_BITMAP_C_
 #include "jemalloc/internal/jemalloc_internal.h"
 
 /******************************************************************************/

src/chunk.c

@@ -4,19 +4,24 @@
 /******************************************************************************/
 /* Data. */
 
-size_t	opt_lg_chunk = LG_CHUNK_DEFAULT;
-#ifdef JEMALLOC_SWAP
-bool	opt_overcommit = true;
-#endif
+const char	*opt_dss = DSS_DEFAULT;
+size_t		opt_lg_chunk = LG_CHUNK_DEFAULT;
 
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
 malloc_mutex_t	chunks_mtx;
 chunk_stats_t	stats_chunks;
-#endif
 
-#ifdef JEMALLOC_IVSALLOC
+/*
+ * Trees of chunks that were previously allocated (trees differ only in node
+ * ordering).  These are used when allocating chunks, in an attempt to re-use
+ * address space.  Depending on function, different tree orderings are needed,
+ * which is why there are two trees with the same contents.
+ */
+static extent_tree_t	chunks_szad_mmap;
+static extent_tree_t	chunks_ad_mmap;
+static extent_tree_t	chunks_szad_dss;
+static extent_tree_t	chunks_ad_dss;
+
 rtree_t		*chunks_rtree;
-#endif
 
 /* Various chunk-related settings. */
 size_t		chunksize;
@@ -26,6 +31,109 @@ size_t		map_bias;
 size_t		arena_maxclass; /* Max size class for arenas. */
 
 /******************************************************************************/
+/* Function prototypes for non-inline static functions. */
+
+static void	*chunk_recycle(extent_tree_t *chunks_szad,
+    extent_tree_t *chunks_ad, size_t size, size_t alignment, bool base,
+    bool *zero);
+static void	chunk_record(extent_tree_t *chunks_szad,
+    extent_tree_t *chunks_ad, void *chunk, size_t size);
+
+/******************************************************************************/
+
+static void *
+chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size,
+    size_t alignment, bool base, bool *zero)
+{
+	void *ret;
+	extent_node_t *node;
+	extent_node_t key;
+	size_t alloc_size, leadsize, trailsize;
+	bool zeroed;
+
+	if (base) {
+		/*
+		 * This function may need to call base_node_{,de}alloc(), but
+		 * the current chunk allocation request is on behalf of the
+		 * base allocator.  Avoid deadlock (and if that weren't an
+		 * issue, potential for infinite recursion) by returning NULL.
+		 */
+		return (NULL);
+	}
+
+	alloc_size = size + alignment - chunksize;
+	/* Beware size_t wrap-around. */
+	if (alloc_size < size)
+		return (NULL);
+	key.addr = NULL;
+	key.size = alloc_size;
+	malloc_mutex_lock(&chunks_mtx);
+	node = extent_tree_szad_nsearch(chunks_szad, &key);
+	if (node == NULL) {
+		malloc_mutex_unlock(&chunks_mtx);
+		return (NULL);
+	}
+	leadsize = ALIGNMENT_CEILING((uintptr_t)node->addr, alignment) -
+	    (uintptr_t)node->addr;
+	assert(node->size >= leadsize + size);
+	trailsize = node->size - leadsize - size;
+	ret = (void *)((uintptr_t)node->addr + leadsize);
+	zeroed = node->zeroed;
+	if (zeroed)
+	    *zero = true;
+	/* Remove node from the tree. */
+	extent_tree_szad_remove(chunks_szad, node);
+	extent_tree_ad_remove(chunks_ad, node);
+	if (leadsize != 0) {
+		/* Insert the leading space as a smaller chunk. */
+		node->size = leadsize;
+		extent_tree_szad_insert(chunks_szad, node);
+		extent_tree_ad_insert(chunks_ad, node);
+		node = NULL;
+	}
+	if (trailsize != 0) {
+		/* Insert the trailing space as a smaller chunk. */
+		if (node == NULL) {
+			/*
+			 * An additional node is required, but
+			 * base_node_alloc() can cause a new base chunk to be
+			 * allocated.  Drop chunks_mtx in order to avoid
+			 * deadlock, and if node allocation fails, deallocate
+			 * the result before returning an error.
+			 */
+			malloc_mutex_unlock(&chunks_mtx);
+			node = base_node_alloc();
+			if (node == NULL) {
+				chunk_dealloc(ret, size, true);
+				return (NULL);
+			}
+			malloc_mutex_lock(&chunks_mtx);
+		}
+		node->addr = (void *)((uintptr_t)(ret) + size);
+		node->size = trailsize;
+		node->zeroed = zeroed;
+		extent_tree_szad_insert(chunks_szad, node);
+		extent_tree_ad_insert(chunks_ad, node);
+		node = NULL;
+	}
+	malloc_mutex_unlock(&chunks_mtx);
+
+	if (node != NULL)
+		base_node_dealloc(node);
+	if (*zero) {
+		if (zeroed == false)
+			memset(ret, 0, size);
+		else if (config_debug) {
+			size_t i;
+			size_t *p = (size_t *)(uintptr_t)ret;
+
+			VALGRIND_MAKE_MEM_DEFINED(ret, size);
+			for (i = 0; i < size / sizeof(size_t); i++)
+				assert(p[i] == 0);
+		}
+	}
+	return (ret);
+}
 
 /*
  * If the caller specifies (*zero == false), it is still possible to receive
@@ -34,81 +142,177 @@ size_t		arena_maxclass; /* Max size class for arenas. */
  * advantage of them if they are returned.
  */
 void *
-chunk_alloc(size_t size, bool base, bool *zero)
+chunk_alloc(size_t size, size_t alignment, bool base, bool *zero,
+    dss_prec_t dss_prec)
 {
 	void *ret;
 
 	assert(size != 0);
 	assert((size & chunksize_mask) == 0);
+	assert(alignment != 0);
+	assert((alignment & chunksize_mask) == 0);
 
-#ifdef JEMALLOC_SWAP
-	if (swap_enabled) {
-		ret = chunk_alloc_swap(size, zero);
-		if (ret != NULL)
-			goto RETURN;
+	/* "primary" dss. */
+	if (config_dss && dss_prec == dss_prec_primary) {
+		if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size,
+		    alignment, base, zero)) != NULL)
+			goto label_return;
+		if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL)
+			goto label_return;
 	}
-
-	if (swap_enabled == false || opt_overcommit) {
-#endif
-#ifdef JEMALLOC_DSS
-		ret = chunk_alloc_dss(size, zero);
-		if (ret != NULL)
-			goto RETURN;
-#endif
-		ret = chunk_alloc_mmap(size);
-		if (ret != NULL) {
-			*zero = true;
-			goto RETURN;
-		}
-#ifdef JEMALLOC_SWAP
+	/* mmap. */
+	if ((ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap, size,
+	    alignment, base, zero)) != NULL)
+		goto label_return;
+	if ((ret = chunk_alloc_mmap(size, alignment, zero)) != NULL)
+		goto label_return;
+	/* "secondary" dss. */
+	if (config_dss && dss_prec == dss_prec_secondary) {
+		if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size,
+		    alignment, base, zero)) != NULL)
+			goto label_return;
+		if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL)
+			goto label_return;
 	}
-#endif
 
 	/* All strategies for allocation failed. */
 	ret = NULL;
-RETURN:
-#ifdef JEMALLOC_IVSALLOC
-	if (base == false && ret != NULL) {
-		if (rtree_set(chunks_rtree, (uintptr_t)ret, ret)) {
-			chunk_dealloc(ret, size);
-			return (NULL);
-		}
-	}
-#endif
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
+label_return:
 	if (ret != NULL) {
-#  ifdef JEMALLOC_PROF
-		bool gdump;
-#  endif
-		malloc_mutex_lock(&chunks_mtx);
-#  ifdef JEMALLOC_STATS
-		stats_chunks.nchunks += (size / chunksize);
-#  endif
-		stats_chunks.curchunks += (size / chunksize);
-		if (stats_chunks.curchunks > stats_chunks.highchunks) {
-			stats_chunks.highchunks = stats_chunks.curchunks;
-#  ifdef JEMALLOC_PROF
-			gdump = true;
-#  endif
+		if (config_ivsalloc && base == false) {
+			if (rtree_set(chunks_rtree, (uintptr_t)ret, 1)) {
+				chunk_dealloc(ret, size, true);
+				return (NULL);
+			}
 		}
-#  ifdef JEMALLOC_PROF
-		else
-			gdump = false;
-#  endif
-		malloc_mutex_unlock(&chunks_mtx);
-#  ifdef JEMALLOC_PROF
-		if (opt_prof && opt_prof_gdump && gdump)
-			prof_gdump();
-#  endif
+		if (config_stats || config_prof) {
+			bool gdump;
+			malloc_mutex_lock(&chunks_mtx);
+			if (config_stats)
+				stats_chunks.nchunks += (size / chunksize);
+			stats_chunks.curchunks += (size / chunksize);
+			if (stats_chunks.curchunks > stats_chunks.highchunks) {
+				stats_chunks.highchunks =
+				    stats_chunks.curchunks;
+				if (config_prof)
+					gdump = true;
+			} else if (config_prof)
+				gdump = false;
+			malloc_mutex_unlock(&chunks_mtx);
+			if (config_prof && opt_prof && opt_prof_gdump && gdump)
+				prof_gdump();
+		}
+		if (config_valgrind)
+			VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 	}
-#endif
-
 	assert(CHUNK_ADDR2BASE(ret) == ret);
 	return (ret);
 }
 
+static void
+chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
+    size_t size)
+{
+	bool unzeroed;
+	extent_node_t *xnode, *node, *prev, *xprev, key;
+
+	unzeroed = pages_purge(chunk, size);
+	VALGRIND_MAKE_MEM_NOACCESS(chunk, size);
+
+	/*
+	 * Allocate a node before acquiring chunks_mtx even though it might not
+	 * be needed, because base_node_alloc() may cause a new base chunk to
+	 * be allocated, which could cause deadlock if chunks_mtx were already
+	 * held.
+	 */
+	xnode = base_node_alloc();
+	/* Use xprev to implement conditional deferred deallocation of prev. */
+	xprev = NULL;
+
+	malloc_mutex_lock(&chunks_mtx);
+	key.addr = (void *)((uintptr_t)chunk + size);
+	node = extent_tree_ad_nsearch(chunks_ad, &key);
+	/* Try to coalesce forward. */
+	if (node != NULL && node->addr == key.addr) {
+		/*
+		 * Coalesce chunk with the following address range.  This does
+		 * not change the position within chunks_ad, so only
+		 * remove/insert from/into chunks_szad.
+		 */
+		extent_tree_szad_remove(chunks_szad, node);
+		node->addr = chunk;
+		node->size += size;
+		node->zeroed = (node->zeroed && (unzeroed == false));
+		extent_tree_szad_insert(chunks_szad, node);
+	} else {
+		/* Coalescing forward failed, so insert a new node. */
+		if (xnode == NULL) {
+			/*
+			 * base_node_alloc() failed, which is an exceedingly
+			 * unlikely failure.  Leak chunk; its pages have
+			 * already been purged, so this is only a virtual
+			 * memory leak.
+			 */
+			goto label_return;
+		}
+		node = xnode;
+		xnode = NULL; /* Prevent deallocation below. */
+		node->addr = chunk;
+		node->size = size;
+		node->zeroed = (unzeroed == false);
+		extent_tree_ad_insert(chunks_ad, node);
+		extent_tree_szad_insert(chunks_szad, node);
+	}
+
+	/* Try to coalesce backward. */
+	prev = extent_tree_ad_prev(chunks_ad, node);
+	if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) ==
+	    chunk) {
+		/*
+		 * Coalesce chunk with the previous address range.  This does
+		 * not change the position within chunks_ad, so only
+		 * remove/insert node from/into chunks_szad.
+		 */
+		extent_tree_szad_remove(chunks_szad, prev);
+		extent_tree_ad_remove(chunks_ad, prev);
+
+		extent_tree_szad_remove(chunks_szad, node);
+		node->addr = prev->addr;
+		node->size += prev->size;
+		node->zeroed = (node->zeroed && prev->zeroed);
+		extent_tree_szad_insert(chunks_szad, node);
+
+		xprev = prev;
+	}
+
+label_return:
+	malloc_mutex_unlock(&chunks_mtx);
+	/*
+	 * Deallocate xnode and/or xprev after unlocking chunks_mtx in order to
+	 * avoid potential deadlock.
+	 */
+	if (xnode != NULL)
+		base_node_dealloc(xnode);
+	if (xprev != NULL)
+		base_node_dealloc(xprev);
+}
+
 void
-chunk_dealloc(void *chunk, size_t size)
+chunk_unmap(void *chunk, size_t size)
+{
+	assert(chunk != NULL);
+	assert(CHUNK_ADDR2BASE(chunk) == chunk);
+	assert(size != 0);
+	assert((size & chunksize_mask) == 0);
+
+	if (config_dss && chunk_in_dss(chunk))
+		chunk_record(&chunks_szad_dss, &chunks_ad_dss, chunk, size);
+	else if (chunk_dealloc_mmap(chunk, size))
+		chunk_record(&chunks_szad_mmap, &chunks_ad_mmap, chunk, size);
+}
+
+void
+chunk_dealloc(void *chunk, size_t size, bool unmap)
 {
 
 	assert(chunk != NULL);
@@ -116,24 +320,17 @@ chunk_dealloc(void *chunk, size_t size)
 	assert(size != 0);
 	assert((size & chunksize_mask) == 0);
 
-#ifdef JEMALLOC_IVSALLOC
-	rtree_set(chunks_rtree, (uintptr_t)chunk, NULL);
-#endif
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-	malloc_mutex_lock(&chunks_mtx);
-	stats_chunks.curchunks -= (size / chunksize);
-	malloc_mutex_unlock(&chunks_mtx);
-#endif
+	if (config_ivsalloc)
+		rtree_set(chunks_rtree, (uintptr_t)chunk, 0);
+	if (config_stats || config_prof) {
+		malloc_mutex_lock(&chunks_mtx);
+		assert(stats_chunks.curchunks >= (size / chunksize));
+		stats_chunks.curchunks -= (size / chunksize);
+		malloc_mutex_unlock(&chunks_mtx);
+	}
 
-#ifdef JEMALLOC_SWAP
-	if (swap_enabled && chunk_dealloc_swap(chunk, size) == false)
-		return;
-#endif
-#ifdef JEMALLOC_DSS
-	if (chunk_dealloc_dss(chunk, size) == false)
-		return;
-#endif
-	chunk_dealloc_mmap(chunk, size);
+	if (unmap)
+		chunk_unmap(chunk, size);
 }
 
 bool
@@ -142,30 +339,57 @@ chunk_boot(void)
 
 	/* Set variables according to the value of opt_lg_chunk. */
 	chunksize = (ZU(1) << opt_lg_chunk);
-	assert(chunksize >= PAGE_SIZE);
+	assert(chunksize >= PAGE);
 	chunksize_mask = chunksize - 1;
-	chunk_npages = (chunksize >> PAGE_SHIFT);
+	chunk_npages = (chunksize >> LG_PAGE);
 
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-	if (malloc_mutex_init(&chunks_mtx))
+	if (config_stats || config_prof) {
+		if (malloc_mutex_init(&chunks_mtx))
+			return (true);
+		memset(&stats_chunks, 0, sizeof(chunk_stats_t));
+	}
+	if (config_dss && chunk_dss_boot())
 		return (true);
-	memset(&stats_chunks, 0, sizeof(chunk_stats_t));
-#endif
-#ifdef JEMALLOC_SWAP
-	if (chunk_swap_boot())
-		return (true);
-#endif
-	if (chunk_mmap_boot())
-		return (true);
-#ifdef JEMALLOC_DSS
-	if (chunk_dss_boot())
-		return (true);
-#endif
-#ifdef JEMALLOC_IVSALLOC
-	chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) - opt_lg_chunk);
-	if (chunks_rtree == NULL)
-		return (true);
-#endif
+	extent_tree_szad_new(&chunks_szad_mmap);
+	extent_tree_ad_new(&chunks_ad_mmap);
+	extent_tree_szad_new(&chunks_szad_dss);
+	extent_tree_ad_new(&chunks_ad_dss);
+	if (config_ivsalloc) {
+		chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) -
+		    opt_lg_chunk, base_alloc, NULL);
+		if (chunks_rtree == NULL)
+			return (true);
+	}
 
 	return (false);
 }
+
+void
+chunk_prefork(void)
+{
+
+	malloc_mutex_prefork(&chunks_mtx);
+	if (config_ivsalloc)
+		rtree_prefork(chunks_rtree);
+	chunk_dss_prefork();
+}
+
+void
+chunk_postfork_parent(void)
+{
+
+	chunk_dss_postfork_parent();
+	if (config_ivsalloc)
+		rtree_postfork_parent(chunks_rtree);
+	malloc_mutex_postfork_parent(&chunks_mtx);
+}
+
+void
+chunk_postfork_child(void)
+{
+
+	chunk_dss_postfork_child();
+	if (config_ivsalloc)
+		rtree_postfork_child(chunks_rtree);
+	malloc_mutex_postfork_child(&chunks_mtx);
+}

src/chunk_dss.c

@@ -1,82 +1,78 @@
 #define	JEMALLOC_CHUNK_DSS_C_
 #include "jemalloc/internal/jemalloc_internal.h"
-#ifdef JEMALLOC_DSS
 /******************************************************************************/
 /* Data. */
 
-malloc_mutex_t	dss_mtx;
+const char	*dss_prec_names[] = {
+	"disabled",
+	"primary",
+	"secondary",
+	"N/A"
+};
 
-/* Base address of the DSS. */
-static void	*dss_base;
-/* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */
-static void	*dss_prev;
-/* Current upper limit on DSS addresses. */
-static void	*dss_max;
+/* Current dss precedence default, used when creating new arenas. */
+static dss_prec_t	dss_prec_default = DSS_PREC_DEFAULT;
 
 /*
- * Trees of chunks that were previously allocated (trees differ only in node
- * ordering).  These are used when allocating chunks, in an attempt to re-use
- * address space.  Depending on function, different tree orderings are needed,
- * which is why there are two trees with the same contents.
+ * Protects sbrk() calls.  This avoids malloc races among threads, though it
+ * does not protect against races with threads that call sbrk() directly.
  */
-static extent_tree_t	dss_chunks_szad;
-static extent_tree_t	dss_chunks_ad;
+static malloc_mutex_t	dss_mtx;
 
-/******************************************************************************/
-/* Function prototypes for non-inline static functions. */
-
-static void	*chunk_recycle_dss(size_t size, bool *zero);
-static extent_node_t *chunk_dealloc_dss_record(void *chunk, size_t size);
+/* Base address of the DSS. */
+static void		*dss_base;
+/* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */
+static void		*dss_prev;
+/* Current upper limit on DSS addresses. */
+static void		*dss_max;
 
 /******************************************************************************/
 
 static void *
-chunk_recycle_dss(size_t size, bool *zero)
+chunk_dss_sbrk(intptr_t increment)
 {
-	extent_node_t *node, key;
-
-	key.addr = NULL;
-	key.size = size;
-	malloc_mutex_lock(&dss_mtx);
-	node = extent_tree_szad_nsearch(&dss_chunks_szad, &key);
-	if (node != NULL) {
-		void *ret = node->addr;
-
-		/* Remove node from the tree. */
-		extent_tree_szad_remove(&dss_chunks_szad, node);
-		if (node->size == size) {
-			extent_tree_ad_remove(&dss_chunks_ad, node);
-			base_node_dealloc(node);
-		} else {
-			/*
-			 * Insert the remainder of node's address range as a
-			 * smaller chunk.  Its position within dss_chunks_ad
-			 * does not change.
-			 */
-			assert(node->size > size);
-			node->addr = (void *)((uintptr_t)node->addr + size);
-			node->size -= size;
-			extent_tree_szad_insert(&dss_chunks_szad, node);
-		}
-		malloc_mutex_unlock(&dss_mtx);
-
-		if (*zero)
-			memset(ret, 0, size);
-		return (ret);
-	}
-	malloc_mutex_unlock(&dss_mtx);
 
+#ifdef JEMALLOC_HAVE_SBRK
+	return (sbrk(increment));
+#else
+	not_implemented();
 	return (NULL);
+#endif
+}
+
+dss_prec_t
+chunk_dss_prec_get(void)
+{
+	dss_prec_t ret;
+
+	if (config_dss == false)
+		return (dss_prec_disabled);
+	malloc_mutex_lock(&dss_mtx);
+	ret = dss_prec_default;
+	malloc_mutex_unlock(&dss_mtx);
+	return (ret);
+}
+
+bool
+chunk_dss_prec_set(dss_prec_t dss_prec)
+{
+
+	if (config_dss == false)
+		return (true);
+	malloc_mutex_lock(&dss_mtx);
+	dss_prec_default = dss_prec;
+	malloc_mutex_unlock(&dss_mtx);
+	return (false);
 }
 
 void *
-chunk_alloc_dss(size_t size, bool *zero)
+chunk_alloc_dss(size_t size, size_t alignment, bool *zero)
 {
 	void *ret;
 
-	ret = chunk_recycle_dss(size, zero);
-	if (ret != NULL)
-		return (ret);
+	cassert(config_dss);
+	assert(size > 0 && (size & chunksize_mask) == 0);
+	assert(alignment > 0 && (alignment & chunksize_mask) == 0);
 
 	/*
 	 * sbrk() uses a signed increment argument, so take care not to
@@ -87,6 +83,8 @@ chunk_alloc_dss(size_t size, bool *zero)
 
 	malloc_mutex_lock(&dss_mtx);
 	if (dss_prev != (void *)-1) {
+		size_t gap_size, cpad_size;
+		void *cpad, *dss_next;
 		intptr_t incr;
 
 		/*
@@ -96,27 +94,41 @@ chunk_alloc_dss(size_t size, bool *zero)
 		 */
 		do {
 			/* Get the current end of the DSS. */
-			dss_max = sbrk(0);
-
+			dss_max = chunk_dss_sbrk(0);
 			/*
 			 * Calculate how much padding is necessary to
 			 * chunk-align the end of the DSS.
 			 */
-			incr = (intptr_t)size
-			    - (intptr_t)CHUNK_ADDR2OFFSET(dss_max);
-			if (incr == (intptr_t)size)
-				ret = dss_max;
-			else {
-				ret = (void *)((intptr_t)dss_max + incr);
-				incr += size;
+			gap_size = (chunksize - CHUNK_ADDR2OFFSET(dss_max)) &
+			    chunksize_mask;
+			/*
+			 * Compute how much chunk-aligned pad space (if any) is
+			 * necessary to satisfy alignment.  This space can be
+			 * recycled for later use.
+			 */
+			cpad = (void *)((uintptr_t)dss_max + gap_size);
+			ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max,
+			    alignment);
+			cpad_size = (uintptr_t)ret - (uintptr_t)cpad;
+			dss_next = (void *)((uintptr_t)ret + size);
+			if ((uintptr_t)ret < (uintptr_t)dss_max ||
+			    (uintptr_t)dss_next < (uintptr_t)dss_max) {
+				/* Wrap-around. */
+				malloc_mutex_unlock(&dss_mtx);
+				return (NULL);
 			}
-
-			dss_prev = sbrk(incr);
+			incr = gap_size + cpad_size + size;
+			dss_prev = chunk_dss_sbrk(incr);
 			if (dss_prev == dss_max) {
 				/* Success. */
-				dss_max = (void *)((intptr_t)dss_prev + incr);
+				dss_max = dss_next;
 				malloc_mutex_unlock(&dss_mtx);
-				*zero = true;
+				if (cpad_size != 0)
+					chunk_unmap(cpad, cpad_size);
+				if (*zero) {
+					VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
+					memset(ret, 0, size);
+				}
 				return (ret);
 			}
 		} while (dss_prev != (void *)-1);
@@ -126,84 +138,13 @@ chunk_alloc_dss(size_t size, bool *zero)
 	return (NULL);
 }
 
-static extent_node_t *
-chunk_dealloc_dss_record(void *chunk, size_t size)
-{
-	extent_node_t *xnode, *node, *prev, key;
-
-	xnode = NULL;
-	while (true) {
-		key.addr = (void *)((uintptr_t)chunk + size);
-		node = extent_tree_ad_nsearch(&dss_chunks_ad, &key);
-		/* Try to coalesce forward. */
-		if (node != NULL && node->addr == key.addr) {
-			/*
-			 * Coalesce chunk with the following address range.
-			 * This does not change the position within
-			 * dss_chunks_ad, so only remove/insert from/into
-			 * dss_chunks_szad.
-			 */
-			extent_tree_szad_remove(&dss_chunks_szad, node);
-			node->addr = chunk;
-			node->size += size;
-			extent_tree_szad_insert(&dss_chunks_szad, node);
-			break;
-		} else if (xnode == NULL) {
-			/*
-			 * It is possible that base_node_alloc() will cause a
-			 * new base chunk to be allocated, so take care not to
-			 * deadlock on dss_mtx, and recover if another thread
-			 * deallocates an adjacent chunk while this one is busy
-			 * allocating xnode.
-			 */
-			malloc_mutex_unlock(&dss_mtx);
-			xnode = base_node_alloc();
-			malloc_mutex_lock(&dss_mtx);
-			if (xnode == NULL)
-				return (NULL);
-		} else {
-			/* Coalescing forward failed, so insert a new node. */
-			node = xnode;
-			xnode = NULL;
-			node->addr = chunk;
-			node->size = size;
-			extent_tree_ad_insert(&dss_chunks_ad, node);
-			extent_tree_szad_insert(&dss_chunks_szad, node);
-			break;
-		}
-	}
-	/* Discard xnode if it ended up unused do to a race. */
-	if (xnode != NULL)
-		base_node_dealloc(xnode);
-
-	/* Try to coalesce backward. */
-	prev = extent_tree_ad_prev(&dss_chunks_ad, node);
-	if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) ==
-	    chunk) {
-		/*
-		 * Coalesce chunk with the previous address range.  This does
-		 * not change the position within dss_chunks_ad, so only
-		 * remove/insert node from/into dss_chunks_szad.
-		 */
-		extent_tree_szad_remove(&dss_chunks_szad, prev);
-		extent_tree_ad_remove(&dss_chunks_ad, prev);
-
-		extent_tree_szad_remove(&dss_chunks_szad, node);
-		node->addr = prev->addr;
-		node->size += prev->size;
-		extent_tree_szad_insert(&dss_chunks_szad, node);
-
-		base_node_dealloc(prev);
-	}
-
-	return (node);
-}
-
 bool
 chunk_in_dss(void *chunk)
 {
 	bool ret;
 
+	cassert(config_dss);
+
 	malloc_mutex_lock(&dss_mtx);
 	if ((uintptr_t)chunk >= (uintptr_t)dss_base
 	    && (uintptr_t)chunk < (uintptr_t)dss_max)
@@ -215,70 +156,43 @@ chunk_in_dss(void *chunk)
 	return (ret);
 }
 
-bool
-chunk_dealloc_dss(void *chunk, size_t size)
-{
-	bool ret;
-
-	malloc_mutex_lock(&dss_mtx);
-	if ((uintptr_t)chunk >= (uintptr_t)dss_base
-	    && (uintptr_t)chunk < (uintptr_t)dss_max) {
-		extent_node_t *node;
-
-		/* Try to coalesce with other unused chunks. */
-		node = chunk_dealloc_dss_record(chunk, size);
-		if (node != NULL) {
-			chunk = node->addr;
-			size = node->size;
-		}
-
-		/* Get the current end of the DSS. */
-		dss_max = sbrk(0);
-
-		/*
-		 * Try to shrink the DSS if this chunk is at the end of the
-		 * DSS.  The sbrk() call here is subject to a race condition
-		 * with threads that use brk(2) or sbrk(2) directly, but the
-		 * alternative would be to leak memory for the sake of poorly
-		 * designed multi-threaded programs.
-		 */
-		if ((void *)((uintptr_t)chunk + size) == dss_max
-		    && (dss_prev = sbrk(-(intptr_t)size)) == dss_max) {
-			/* Success. */
-			dss_max = (void *)((intptr_t)dss_prev - (intptr_t)size);
-
-			if (node != NULL) {
-				extent_tree_szad_remove(&dss_chunks_szad, node);
-				extent_tree_ad_remove(&dss_chunks_ad, node);
-				base_node_dealloc(node);
-			}
-		} else
-			madvise(chunk, size, MADV_DONTNEED);
-
-		ret = false;
-		goto RETURN;
-	}
-
-	ret = true;
-RETURN:
-	malloc_mutex_unlock(&dss_mtx);
-	return (ret);
-}
-
 bool
 chunk_dss_boot(void)
 {
 
+	cassert(config_dss);
+
 	if (malloc_mutex_init(&dss_mtx))
 		return (true);
-	dss_base = sbrk(0);
+	dss_base = chunk_dss_sbrk(0);
 	dss_prev = dss_base;
 	dss_max = dss_base;
-	extent_tree_szad_new(&dss_chunks_szad);
-	extent_tree_ad_new(&dss_chunks_ad);
 
 	return (false);
 }
 
+void
+chunk_dss_prefork(void)
+{
+
+	if (config_dss)
+		malloc_mutex_prefork(&dss_mtx);
+}
+
+void
+chunk_dss_postfork_parent(void)
+{
+
+	if (config_dss)
+		malloc_mutex_postfork_parent(&dss_mtx);
+}
+
+void
+chunk_dss_postfork_child(void)
+{
+
+	if (config_dss)
+		malloc_mutex_postfork_child(&dss_mtx);
+}
+
 /******************************************************************************/
-#endif /* JEMALLOC_DSS */

src/chunk_mmap.c

@@ -1,54 +1,37 @@
 #define	JEMALLOC_CHUNK_MMAP_C_
 #include "jemalloc/internal/jemalloc_internal.h"
 
-/******************************************************************************/
-/* Data. */
-
-/*
- * Used by chunk_alloc_mmap() to decide whether to attempt the fast path and
- * potentially avoid some system calls.
- */
-#ifndef NO_TLS
-static __thread bool	mmap_unaligned_tls
-    JEMALLOC_ATTR(tls_model("initial-exec"));
-#define	MMAP_UNALIGNED_GET()	mmap_unaligned_tls
-#define	MMAP_UNALIGNED_SET(v)	do {					\
-	mmap_unaligned_tls = (v);					\
-} while (0)
-#else
-static pthread_key_t	mmap_unaligned_tsd;
-#define	MMAP_UNALIGNED_GET()	((bool)pthread_getspecific(mmap_unaligned_tsd))
-#define	MMAP_UNALIGNED_SET(v)	do {					\
-	pthread_setspecific(mmap_unaligned_tsd, (void *)(v));		\
-} while (0)
-#endif
-
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
 
-static void	*pages_map(void *addr, size_t size, bool noreserve);
+static void	*pages_map(void *addr, size_t size);
 static void	pages_unmap(void *addr, size_t size);
-static void	*chunk_alloc_mmap_slow(size_t size, bool unaligned,
-    bool noreserve);
-static void	*chunk_alloc_mmap_internal(size_t size, bool noreserve);
+static void	*chunk_alloc_mmap_slow(size_t size, size_t alignment,
+    bool *zero);
 
 /******************************************************************************/
 
 static void *
-pages_map(void *addr, size_t size, bool noreserve)
+pages_map(void *addr, size_t size)
 {
 	void *ret;
 
+	assert(size != 0);
+
+#ifdef _WIN32
+	/*
+	 * If VirtualAlloc can't allocate at the given address when one is
+	 * given, it fails and returns NULL.
+	 */
+	ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
+	    PAGE_READWRITE);
+#else
 	/*
 	 * We don't use MAP_FIXED here, because it can cause the *replacement*
 	 * of existing mappings, and we only want to create new mappings.
 	 */
-	int flags = MAP_PRIVATE | MAP_ANON;
-#ifdef MAP_NORESERVE
-	if (noreserve)
-		flags |= MAP_NORESERVE;
-#endif
-	ret = mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0);
+	ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON,
+	    -1, 0);
 	assert(ret != NULL);
 
 	if (ret == MAP_FAILED)
@@ -60,16 +43,15 @@ pages_map(void *addr, size_t size, bool noreserve)
 		if (munmap(ret, size) == -1) {
 			char buf[BUFERROR_BUF];
 
-			buferror(errno, buf, sizeof(buf));
-			malloc_write("<jemalloc>: Error in munmap(): ");
-			malloc_write(buf);
-			malloc_write("\n");
+			buferror(get_errno(), buf, sizeof(buf));
+			malloc_printf("<jemalloc: Error in munmap(): %s\n",
+			    buf);
 			if (opt_abort)
 				abort();
 		}
 		ret = NULL;
 	}
-
+#endif
 	assert(ret == NULL || (addr == NULL && ret != addr)
 	    || (addr != NULL && ret == addr));
 	return (ret);
@@ -79,161 +61,150 @@ static void
 pages_unmap(void *addr, size_t size)
 {
 
-	if (munmap(addr, size) == -1) {
+#ifdef _WIN32
+	if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
+#else
+	if (munmap(addr, size) == -1)
+#endif
+	{
 		char buf[BUFERROR_BUF];
 
-		buferror(errno, buf, sizeof(buf));
-		malloc_write("<jemalloc>: Error in munmap(): ");
-		malloc_write(buf);
-		malloc_write("\n");
+		buferror(get_errno(), buf, sizeof(buf));
+		malloc_printf("<jemalloc>: Error in "
+#ifdef _WIN32
+		              "VirtualFree"
+#else
+		              "munmap"
+#endif
+		              "(): %s\n", buf);
 		if (opt_abort)
 			abort();
 	}
 }
 
 static void *
-chunk_alloc_mmap_slow(size_t size, bool unaligned, bool noreserve)
+pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size)
 {
-	void *ret;
-	size_t offset;
+	void *ret = (void *)((uintptr_t)addr + leadsize);
 
-	/* Beware size_t wrap-around. */
-	if (size + chunksize <= size)
+	assert(alloc_size >= leadsize + size);
+#ifdef _WIN32
+	{
+		void *new_addr;
+
+		pages_unmap(addr, alloc_size);
+		new_addr = pages_map(ret, size);
+		if (new_addr == ret)
+			return (ret);
+		if (new_addr)
+			pages_unmap(new_addr, size);
 		return (NULL);
-
-	ret = pages_map(NULL, size + chunksize, noreserve);
-	if (ret == NULL)
-		return (NULL);
-
-	/* Clean up unneeded leading/trailing space. */
-	offset = CHUNK_ADDR2OFFSET(ret);
-	if (offset != 0) {
-		/* Note that mmap() returned an unaligned mapping. */
-		unaligned = true;
-
-		/* Leading space. */
-		pages_unmap(ret, chunksize - offset);
-
-		ret = (void *)((uintptr_t)ret +
-		    (chunksize - offset));
-
-		/* Trailing space. */
-		pages_unmap((void *)((uintptr_t)ret + size),
-		    offset);
-	} else {
-		/* Trailing space only. */
-		pages_unmap((void *)((uintptr_t)ret + size),
-		    chunksize);
 	}
+#else
+	{
+		size_t trailsize = alloc_size - leadsize - size;
 
-	/*
-	 * If mmap() returned an aligned mapping, reset mmap_unaligned so that
-	 * the next chunk_alloc_mmap() execution tries the fast allocation
-	 * method.
-	 */
-	if (unaligned == false)
-		MMAP_UNALIGNED_SET(false);
+		if (leadsize != 0)
+			pages_unmap(addr, leadsize);
+		if (trailsize != 0)
+			pages_unmap((void *)((uintptr_t)ret + size), trailsize);
+		return (ret);
+	}
+#endif
+}
 
-	return (ret);
+bool
+pages_purge(void *addr, size_t length)
+{
+	bool unzeroed;
+
+#ifdef _WIN32
+	VirtualAlloc(addr, length, MEM_RESET, PAGE_READWRITE);
+	unzeroed = true;
+#else
+#  ifdef JEMALLOC_PURGE_MADVISE_DONTNEED
+#    define JEMALLOC_MADV_PURGE MADV_DONTNEED
+#    define JEMALLOC_MADV_ZEROS true
+#  elif defined(JEMALLOC_PURGE_MADVISE_FREE)
+#    define JEMALLOC_MADV_PURGE MADV_FREE
+#    define JEMALLOC_MADV_ZEROS false
+#  else
+#    error "No method defined for purging unused dirty pages."
+#  endif
+	int err = madvise(addr, length, JEMALLOC_MADV_PURGE);
+	unzeroed = (JEMALLOC_MADV_ZEROS == false || err != 0);
+#  undef JEMALLOC_MADV_PURGE
+#  undef JEMALLOC_MADV_ZEROS
+#endif
+	return (unzeroed);
 }
 
 static void *
-chunk_alloc_mmap_internal(size_t size, bool noreserve)
+chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero)
+{
+	void *ret, *pages;
+	size_t alloc_size, leadsize;
+
+	alloc_size = size + alignment - PAGE;
+	/* Beware size_t wrap-around. */
+	if (alloc_size < size)
+		return (NULL);
+	do {
+		pages = pages_map(NULL, alloc_size);
+		if (pages == NULL)
+			return (NULL);
+		leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) -
+		    (uintptr_t)pages;
+		ret = pages_trim(pages, alloc_size, leadsize, size);
+	} while (ret == NULL);
+
+	assert(ret != NULL);
+	*zero = true;
+	return (ret);
+}
+
+void *
+chunk_alloc_mmap(size_t size, size_t alignment, bool *zero)
 {
 	void *ret;
+	size_t offset;
 
 	/*
 	 * Ideally, there would be a way to specify alignment to mmap() (like
 	 * NetBSD has), but in the absence of such a feature, we have to work
 	 * hard to efficiently create aligned mappings.  The reliable, but
 	 * slow method is to create a mapping that is over-sized, then trim the
-	 * excess.  However, that always results in at least one call to
+	 * excess.  However, that always results in one or two calls to
 	 * pages_unmap().
 	 *
-	 * A more optimistic approach is to try mapping precisely the right
-	 * amount, then try to append another mapping if alignment is off.  In
-	 * practice, this works out well as long as the application is not
-	 * interleaving mappings via direct mmap() calls.  If we do run into a
-	 * situation where there is an interleaved mapping and we are unable to
-	 * extend an unaligned mapping, our best option is to switch to the
-	 * slow method until mmap() returns another aligned mapping.  This will
-	 * tend to leave a gap in the memory map that is too small to cause
-	 * later problems for the optimistic method.
-	 *
-	 * Another possible confounding factor is address space layout
-	 * randomization (ASLR), which causes mmap(2) to disregard the
-	 * requested address.  mmap_unaligned tracks whether the previous
-	 * chunk_alloc_mmap() execution received any unaligned or relocated
-	 * mappings, and if so, the current execution will immediately fall
-	 * back to the slow method.  However, we keep track of whether the fast
-	 * method would have succeeded, and if so, we make a note to try the
-	 * fast method next time.
+	 * Optimistically try mapping precisely the right amount before falling
+	 * back to the slow method, with the expectation that the optimistic
+	 * approach works most of the time.
 	 */
 
-	if (MMAP_UNALIGNED_GET() == false) {
-		size_t offset;
+	assert(alignment != 0);
+	assert((alignment & chunksize_mask) == 0);
 
-		ret = pages_map(NULL, size, noreserve);
-		if (ret == NULL)
-			return (NULL);
-
-		offset = CHUNK_ADDR2OFFSET(ret);
-		if (offset != 0) {
-			MMAP_UNALIGNED_SET(true);
-			/* Try to extend chunk boundary. */
-			if (pages_map((void *)((uintptr_t)ret + size),
-			    chunksize - offset, noreserve) == NULL) {
-				/*
-				 * Extension failed.  Clean up, then revert to
-				 * the reliable-but-expensive method.
-				 */
-				pages_unmap(ret, size);
-				ret = chunk_alloc_mmap_slow(size, true,
-				    noreserve);
-			} else {
-				/* Clean up unneeded leading space. */
-				pages_unmap(ret, chunksize - offset);
-				ret = (void *)((uintptr_t)ret + (chunksize -
-				    offset));
-			}
-		}
-	} else
-		ret = chunk_alloc_mmap_slow(size, false, noreserve);
+	ret = pages_map(NULL, size);
+	if (ret == NULL)
+		return (NULL);
+	offset = ALIGNMENT_ADDR2OFFSET(ret, alignment);
+	if (offset != 0) {
+		pages_unmap(ret, size);
+		return (chunk_alloc_mmap_slow(size, alignment, zero));
+	}
 
+	assert(ret != NULL);
+	*zero = true;
 	return (ret);
 }
 
-void *
-chunk_alloc_mmap(size_t size)
-{
-
-	return (chunk_alloc_mmap_internal(size, false));
-}
-
-void *
-chunk_alloc_mmap_noreserve(size_t size)
-{
-
-	return (chunk_alloc_mmap_internal(size, true));
-}
-
-void
+bool
 chunk_dealloc_mmap(void *chunk, size_t size)
 {
 
-	pages_unmap(chunk, size);
-}
-
-bool
-chunk_mmap_boot(void)
-{
-
-#ifdef NO_TLS
-	if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) {
-		malloc_write("<jemalloc>: Error in pthread_key_create()\n");
-		return (true);
-	}
-#endif
-
-	return (false);
+	if (config_munmap)
+		pages_unmap(chunk, size);
+
+	return (config_munmap == false);
 }

src/chunk_swap.c

@@ -1,402 +0,0 @@
-#define	JEMALLOC_CHUNK_SWAP_C_
-#include "jemalloc/internal/jemalloc_internal.h"
-#ifdef JEMALLOC_SWAP
-/******************************************************************************/
-/* Data. */
-
-malloc_mutex_t	swap_mtx;
-bool		swap_enabled;
-bool		swap_prezeroed;
-size_t		swap_nfds;
-int		*swap_fds;
-#ifdef JEMALLOC_STATS
-size_t		swap_avail;
-#endif
-
-/* Base address of the mmap()ed file(s). */
-static void	*swap_base;
-/* Current end of the space in use (<= swap_max). */
-static void	*swap_end;
-/* Absolute upper limit on file-backed addresses. */
-static void	*swap_max;
-
-/*
- * Trees of chunks that were previously allocated (trees differ only in node
- * ordering).  These are used when allocating chunks, in an attempt to re-use
- * address space.  Depending on function, different tree orderings are needed,
- * which is why there are two trees with the same contents.
- */
-static extent_tree_t	swap_chunks_szad;
-static extent_tree_t	swap_chunks_ad;
-
-/******************************************************************************/
-/* Function prototypes for non-inline static functions. */
-
-static void	*chunk_recycle_swap(size_t size, bool *zero);
-static extent_node_t *chunk_dealloc_swap_record(void *chunk, size_t size);
-
-/******************************************************************************/
-
-static void *
-chunk_recycle_swap(size_t size, bool *zero)
-{
-	extent_node_t *node, key;
-
-	key.addr = NULL;
-	key.size = size;
-	malloc_mutex_lock(&swap_mtx);
-	node = extent_tree_szad_nsearch(&swap_chunks_szad, &key);
-	if (node != NULL) {
-		void *ret = node->addr;
-
-		/* Remove node from the tree. */
-		extent_tree_szad_remove(&swap_chunks_szad, node);
-		if (node->size == size) {
-			extent_tree_ad_remove(&swap_chunks_ad, node);
-			base_node_dealloc(node);
-		} else {
-			/*
-			 * Insert the remainder of node's address range as a
-			 * smaller chunk.  Its position within swap_chunks_ad
-			 * does not change.
-			 */
-			assert(node->size > size);
-			node->addr = (void *)((uintptr_t)node->addr + size);
-			node->size -= size;
-			extent_tree_szad_insert(&swap_chunks_szad, node);
-		}
-#ifdef JEMALLOC_STATS
-		swap_avail -= size;
-#endif
-		malloc_mutex_unlock(&swap_mtx);
-
-		if (*zero)
-			memset(ret, 0, size);
-		return (ret);
-	}
-	malloc_mutex_unlock(&swap_mtx);
-
-	return (NULL);
-}
-
-void *
-chunk_alloc_swap(size_t size, bool *zero)
-{
-	void *ret;
-
-	assert(swap_enabled);
-
-	ret = chunk_recycle_swap(size, zero);
-	if (ret != NULL)
-		return (ret);
-
-	malloc_mutex_lock(&swap_mtx);
-	if ((uintptr_t)swap_end + size <= (uintptr_t)swap_max) {
-		ret = swap_end;
-		swap_end = (void *)((uintptr_t)swap_end + size);
-#ifdef JEMALLOC_STATS
-		swap_avail -= size;
-#endif
-		malloc_mutex_unlock(&swap_mtx);
-
-		if (swap_prezeroed)
-			*zero = true;
-		else if (*zero)
-			memset(ret, 0, size);
-	} else {
-		malloc_mutex_unlock(&swap_mtx);
-		return (NULL);
-	}
-
-	return (ret);
-}
-
-static extent_node_t *
-chunk_dealloc_swap_record(void *chunk, size_t size)
-{
-	extent_node_t *xnode, *node, *prev, key;
-
-	xnode = NULL;
-	while (true) {
-		key.addr = (void *)((uintptr_t)chunk + size);
-		node = extent_tree_ad_nsearch(&swap_chunks_ad, &key);
-		/* Try to coalesce forward. */
-		if (node != NULL && node->addr == key.addr) {
-			/*
-			 * Coalesce chunk with the following address range.
-			 * This does not change the position within
-			 * swap_chunks_ad, so only remove/insert from/into
-			 * swap_chunks_szad.
-			 */
-			extent_tree_szad_remove(&swap_chunks_szad, node);
-			node->addr = chunk;
-			node->size += size;
-			extent_tree_szad_insert(&swap_chunks_szad, node);
-			break;
-		} else if (xnode == NULL) {
-			/*
-			 * It is possible that base_node_alloc() will cause a
-			 * new base chunk to be allocated, so take care not to
-			 * deadlock on swap_mtx, and recover if another thread
-			 * deallocates an adjacent chunk while this one is busy
-			 * allocating xnode.
-			 */
-			malloc_mutex_unlock(&swap_mtx);
-			xnode = base_node_alloc();
-			malloc_mutex_lock(&swap_mtx);
-			if (xnode == NULL)
-				return (NULL);
-		} else {
-			/* Coalescing forward failed, so insert a new node. */
-			node = xnode;
-			xnode = NULL;
-			node->addr = chunk;
-			node->size = size;
-			extent_tree_ad_insert(&swap_chunks_ad, node);
-			extent_tree_szad_insert(&swap_chunks_szad, node);
-			break;
-		}
-	}
-	/* Discard xnode if it ended up unused do to a race. */
-	if (xnode != NULL)
-		base_node_dealloc(xnode);
-
-	/* Try to coalesce backward. */
-	prev = extent_tree_ad_prev(&swap_chunks_ad, node);
-	if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) ==
-	    chunk) {
-		/*
-		 * Coalesce chunk with the previous address range.  This does
-		 * not change the position within swap_chunks_ad, so only
-		 * remove/insert node from/into swap_chunks_szad.
-		 */
-		extent_tree_szad_remove(&swap_chunks_szad, prev);
-		extent_tree_ad_remove(&swap_chunks_ad, prev);
-
-		extent_tree_szad_remove(&swap_chunks_szad, node);
-		node->addr = prev->addr;
-		node->size += prev->size;
-		extent_tree_szad_insert(&swap_chunks_szad, node);
-
-		base_node_dealloc(prev);
-	}
-
-	return (node);
-}
-
-bool
-chunk_in_swap(void *chunk)
-{
-	bool ret;
-
-	assert(swap_enabled);
-
-	malloc_mutex_lock(&swap_mtx);
-	if ((uintptr_t)chunk >= (uintptr_t)swap_base
-	    && (uintptr_t)chunk < (uintptr_t)swap_max)
-		ret = true;
-	else
-		ret = false;
-	malloc_mutex_unlock(&swap_mtx);
-
-	return (ret);
-}
-
-bool
-chunk_dealloc_swap(void *chunk, size_t size)
-{
-	bool ret;
-
-	assert(swap_enabled);
-
-	malloc_mutex_lock(&swap_mtx);
-	if ((uintptr_t)chunk >= (uintptr_t)swap_base
-	    && (uintptr_t)chunk < (uintptr_t)swap_max) {
-		extent_node_t *node;
-
-		/* Try to coalesce with other unused chunks. */
-		node = chunk_dealloc_swap_record(chunk, size);
-		if (node != NULL) {
-			chunk = node->addr;
-			size = node->size;
-		}
-
-		/*
-		 * Try to shrink the in-use memory if this chunk is at the end
-		 * of the in-use memory.
-		 */
-		if ((void *)((uintptr_t)chunk + size) == swap_end) {
-			swap_end = (void *)((uintptr_t)swap_end - size);
-
-			if (node != NULL) {
-				extent_tree_szad_remove(&swap_chunks_szad,
-				    node);
-				extent_tree_ad_remove(&swap_chunks_ad, node);
-				base_node_dealloc(node);
-			}
-		} else
-			madvise(chunk, size, MADV_DONTNEED);
-
-#ifdef JEMALLOC_STATS
-		swap_avail += size;
-#endif
-		ret = false;
-		goto RETURN;
-	}
-
-	ret = true;
-RETURN:
-	malloc_mutex_unlock(&swap_mtx);
-	return (ret);
-}
-
-bool
-chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed)
-{
-	bool ret;
-	unsigned i;
-	off_t off;
-	void *vaddr;
-	size_t cumsize, voff;
-	size_t sizes[nfds];
-
-	malloc_mutex_lock(&swap_mtx);
-
-	/* Get file sizes. */
-	for (i = 0, cumsize = 0; i < nfds; i++) {
-		off = lseek(fds[i], 0, SEEK_END);
-		if (off == ((off_t)-1)) {
-			ret = true;
-			goto RETURN;
-		}
-		if (PAGE_CEILING(off) != off) {
-			/* Truncate to a multiple of the page size. */
-			off &= ~PAGE_MASK;
-			if (ftruncate(fds[i], off) != 0) {
-				ret = true;
-				goto RETURN;
-			}
-		}
-		sizes[i] = off;
-		if (cumsize + off < cumsize) {
-			/*
-			 * Cumulative file size is greater than the total
-			 * address space.  Bail out while it's still obvious
-			 * what the problem is.
-			 */
-			ret = true;
-			goto RETURN;
-		}
-		cumsize += off;
-	}
-
-	/* Round down to a multiple of the chunk size. */
-	cumsize &= ~chunksize_mask;
-	if (cumsize == 0) {
-		ret = true;
-		goto RETURN;
-	}
-
-	/*
-	 * Allocate a chunk-aligned region of anonymous memory, which will
-	 * be the final location for the memory-mapped files.
-	 */
-	vaddr = chunk_alloc_mmap_noreserve(cumsize);
-	if (vaddr == NULL) {
-		ret = true;
-		goto RETURN;
-	}
-
-	/* Overlay the files onto the anonymous mapping. */
-	for (i = 0, voff = 0; i < nfds; i++) {
-		void *addr = mmap((void *)((uintptr_t)vaddr + voff), sizes[i],
-		    PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fds[i], 0);
-		if (addr == MAP_FAILED) {
-			char buf[BUFERROR_BUF];
-
-
-			buferror(errno, buf, sizeof(buf));
-			malloc_write(
-			    "<jemalloc>: Error in mmap(..., MAP_FIXED, ...): ");
-			malloc_write(buf);
-			malloc_write("\n");
-			if (opt_abort)
-				abort();
-			if (munmap(vaddr, voff) == -1) {
-				buferror(errno, buf, sizeof(buf));
-				malloc_write("<jemalloc>: Error in munmap(): ");
-				malloc_write(buf);
-				malloc_write("\n");
-			}
-			ret = true;
-			goto RETURN;
-		}
-		assert(addr == (void *)((uintptr_t)vaddr + voff));
-
-		/*
-		 * Tell the kernel that the mapping will be accessed randomly,
-		 * and that it should not gratuitously sync pages to the
-		 * filesystem.
-		 */
-#ifdef MADV_RANDOM
-		madvise(addr, sizes[i], MADV_RANDOM);
-#endif
-#ifdef MADV_NOSYNC
-		madvise(addr, sizes[i], MADV_NOSYNC);
-#endif
-
-		voff += sizes[i];
-	}
-
-	swap_prezeroed = prezeroed;
-	swap_base = vaddr;
-	swap_end = swap_base;
-	swap_max = (void *)((uintptr_t)vaddr + cumsize);
-
-	/* Copy the fds array for mallctl purposes. */
-	swap_fds = (int *)base_alloc(nfds * sizeof(int));
-	if (swap_fds == NULL) {
-		ret = true;
-		goto RETURN;
-	}
-	memcpy(swap_fds, fds, nfds * sizeof(int));
-	swap_nfds = nfds;
-
-#ifdef JEMALLOC_STATS
-	swap_avail = cumsize;
-#endif
-
-	swap_enabled = true;
-
-	ret = false;
-RETURN:
-	malloc_mutex_unlock(&swap_mtx);
-	return (ret);
-}
-
-bool
-chunk_swap_boot(void)
-{
-
-	if (malloc_mutex_init(&swap_mtx))
-		return (true);
-
-	swap_enabled = false;
-	swap_prezeroed = false; /* swap.* mallctl's depend on this. */
-	swap_nfds = 0;
-	swap_fds = NULL;
-#ifdef JEMALLOC_STATS
-	swap_avail = 0;
-#endif
-	swap_base = NULL;
-	swap_end = NULL;
-	swap_max = NULL;
-
-	extent_tree_szad_new(&swap_chunks_szad);
-	extent_tree_ad_new(&swap_chunks_ad);
-
-	return (false);
-}
-
-/******************************************************************************/
-#endif /* JEMALLOC_SWAP */

src/ckh.c

@@ -49,7 +49,7 @@ static void	ckh_shrink(ckh_t *ckh);
  * Search bucket for key and return the cell number if found; SIZE_T_MAX
  * otherwise.
  */
-JEMALLOC_INLINE size_t
+JEMALLOC_INLINE_C size_t
 ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key)
 {
 	ckhc_t *cell;
@@ -67,29 +67,28 @@ ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key)
 /*
  * Search table for key and return cell number if found; SIZE_T_MAX otherwise.
  */
-JEMALLOC_INLINE size_t
+JEMALLOC_INLINE_C size_t
 ckh_isearch(ckh_t *ckh, const void *key)
 {
-	size_t hash1, hash2, bucket, cell;
+	size_t hashes[2], bucket, cell;
 
 	assert(ckh != NULL);
-	dassert(ckh->magic == CKH_MAGIC);
 
-	ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2);
+	ckh->hash(key, hashes);
 
 	/* Search primary bucket. */
-	bucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1);
+	bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1);
 	cell = ckh_bucket_search(ckh, bucket, key);
 	if (cell != SIZE_T_MAX)
 		return (cell);
 
 	/* Search secondary bucket. */
-	bucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1);
+	bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1);
 	cell = ckh_bucket_search(ckh, bucket, key);
 	return (cell);
 }
 
-JEMALLOC_INLINE bool
+JEMALLOC_INLINE_C bool
 ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key,
     const void *data)
 {
@@ -100,7 +99,7 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key,
 	 * Cycle through the cells in the bucket, starting at a random position.
 	 * The randomness avoids worst-case search overhead as buckets fill up.
 	 */
-	prn32(offset, LG_CKH_BUCKET_CELLS, ckh->prn_state, CKH_A, CKH_C);
+	prng32(offset, LG_CKH_BUCKET_CELLS, ckh->prng_state, CKH_A, CKH_C);
 	for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) {
 		cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) +
 		    ((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))];
@@ -121,13 +120,13 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key,
  * eviction/relocation procedure until either success or detection of an
  * eviction/relocation bucket cycle.
  */
-JEMALLOC_INLINE bool
+JEMALLOC_INLINE_C bool
 ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
     void const **argdata)
 {
 	const void *key, *data, *tkey, *tdata;
 	ckhc_t *cell;
-	size_t hash1, hash2, bucket, tbucket;
+	size_t hashes[2], bucket, tbucket;
 	unsigned i;
 
 	bucket = argbucket;
@@ -142,7 +141,7 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
 		 * were an item for which both hashes indicated the same
 		 * bucket.
 		 */
-		prn32(i, LG_CKH_BUCKET_CELLS, ckh->prn_state, CKH_A, CKH_C);
+		prng32(i, LG_CKH_BUCKET_CELLS, ckh->prng_state, CKH_A, CKH_C);
 		cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i];
 		assert(cell->key != NULL);
 
@@ -156,10 +155,11 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
 #endif
 
 		/* Find the alternate bucket for the evicted item. */
-		ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2);
-		tbucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1);
+		ckh->hash(key, hashes);
+		tbucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1);
 		if (tbucket == bucket) {
-			tbucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1);
+			tbucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets)
+			    - 1);
 			/*
 			 * It may be that (tbucket == bucket) still, if the
 			 * item's hashes both indicate this bucket.  However,
@@ -190,22 +190,22 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
 	}
 }
 
-JEMALLOC_INLINE bool
+JEMALLOC_INLINE_C bool
 ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata)
 {
-	size_t hash1, hash2, bucket;
+	size_t hashes[2], bucket;
 	const void *key = *argkey;
 	const void *data = *argdata;
 
-	ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2);
+	ckh->hash(key, hashes);
 
 	/* Try to insert in primary bucket. */
-	bucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1);
+	bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1);
 	if (ckh_try_bucket_insert(ckh, bucket, key, data) == false)
 		return (false);
 
 	/* Try to insert in secondary bucket. */
-	bucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1);
+	bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1);
 	if (ckh_try_bucket_insert(ckh, bucket, key, data) == false)
 		return (false);
 
@@ -219,7 +219,7 @@ ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata)
  * Try to rebuild the hash table from scratch by inserting all items from the
  * old table into the new.
  */
-JEMALLOC_INLINE bool
+JEMALLOC_INLINE_C bool
 ckh_rebuild(ckh_t *ckh, ckhc_t *aTab)
 {
 	size_t count, i, nins;
@@ -265,15 +265,15 @@ ckh_grow(ckh_t *ckh)
 		size_t usize;
 
 		lg_curcells++;
-		usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE, NULL);
+		usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
 		if (usize == 0) {
 			ret = true;
-			goto RETURN;
+			goto label_return;
 		}
 		tab = (ckhc_t *)ipalloc(usize, CACHELINE, true);
 		if (tab == NULL) {
 			ret = true;
-			goto RETURN;
+			goto label_return;
 		}
 		/* Swap in new table. */
 		ttab = ckh->tab;
@@ -293,7 +293,7 @@ ckh_grow(ckh_t *ckh)
 	}
 
 	ret = false;
-RETURN:
+label_return:
 	return (ret);
 }
 
@@ -310,7 +310,7 @@ ckh_shrink(ckh_t *ckh)
 	 */
 	lg_prevbuckets = ckh->lg_curbuckets;
 	lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1;
-	usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE, NULL);
+	usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
 	if (usize == 0)
 		return;
 	tab = (ckhc_t *)ipalloc(usize, CACHELINE, true);
@@ -362,7 +362,7 @@ ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp)
 	ckh->ninserts = 0;
 	ckh->nrelocs = 0;
 #endif
-	ckh->prn_state = 42; /* Value doesn't really matter. */
+	ckh->prng_state = 42; /* Value doesn't really matter. */
 	ckh->count = 0;
 
 	/*
@@ -383,23 +383,19 @@ ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp)
 	ckh->hash = hash;
 	ckh->keycomp = keycomp;
 
-	usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE, NULL);
+	usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE);
 	if (usize == 0) {
 		ret = true;
-		goto RETURN;
+		goto label_return;
 	}
 	ckh->tab = (ckhc_t *)ipalloc(usize, CACHELINE, true);
 	if (ckh->tab == NULL) {
 		ret = true;
-		goto RETURN;
+		goto label_return;
 	}
 
-#ifdef JEMALLOC_DEBUG
-	ckh->magic = CKH_MAGIC;
-#endif
-
 	ret = false;
-RETURN:
+label_return:
 	return (ret);
 }
 
@@ -408,7 +404,6 @@ ckh_delete(ckh_t *ckh)
 {
 
 	assert(ckh != NULL);
-	dassert(ckh->magic == CKH_MAGIC);
 
 #ifdef CKH_VERBOSE
 	malloc_printf(
@@ -423,9 +418,8 @@ ckh_delete(ckh_t *ckh)
 #endif
 
 	idalloc(ckh->tab);
-#ifdef JEMALLOC_DEBUG
-	memset(ckh, 0x5a, sizeof(ckh_t));
-#endif
+	if (config_debug)
+		memset(ckh, 0x5a, sizeof(ckh_t));
 }
 
 size_t
@@ -433,7 +427,6 @@ ckh_count(ckh_t *ckh)
 {
 
 	assert(ckh != NULL);
-	dassert(ckh->magic == CKH_MAGIC);
 
 	return (ckh->count);
 }
@@ -464,7 +457,6 @@ ckh_insert(ckh_t *ckh, const void *key, const void *data)
 	bool ret;
 
 	assert(ckh != NULL);
-	dassert(ckh->magic == CKH_MAGIC);
 	assert(ckh_search(ckh, key, NULL, NULL));
 
 #ifdef CKH_COUNT
@@ -474,12 +466,12 @@ ckh_insert(ckh_t *ckh, const void *key, const void *data)
 	while (ckh_try_insert(ckh, &key, &data)) {
 		if (ckh_grow(ckh)) {
 			ret = true;
-			goto RETURN;
+			goto label_return;
 		}
 	}
 
 	ret = false;
-RETURN:
+label_return:
 	return (ret);
 }
 
@@ -489,7 +481,6 @@ ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data)
 	size_t cell;
 
 	assert(ckh != NULL);
-	dassert(ckh->magic == CKH_MAGIC);
 
 	cell = ckh_isearch(ckh, searchkey);
 	if (cell != SIZE_T_MAX) {
@@ -521,7 +512,6 @@ ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data)
 	size_t cell;
 
 	assert(ckh != NULL);
-	dassert(ckh->magic == CKH_MAGIC);
 
 	cell = ckh_isearch(ckh, searchkey);
 	if (cell != SIZE_T_MAX) {
@@ -536,31 +526,10 @@ ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data)
 }
 
 void
-ckh_string_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2)
+ckh_string_hash(const void *key, size_t r_hash[2])
 {
-	size_t ret1, ret2;
-	uint64_t h;
 
-	assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64));
-	assert(hash1 != NULL);
-	assert(hash2 != NULL);
-
-	h = hash(key, strlen((const char *)key), 0x94122f335b332aeaLLU);
-	if (minbits <= 32) {
-		/*
-		 * Avoid doing multiple hashes, since a single hash provides
-		 * enough bits.
-		 */
-		ret1 = h & ZU(0xffffffffU);
-		ret2 = h >> 32;
-	} else {
-		ret1 = h;
-		ret2 = hash(key, strlen((const char *)key),
-		    0x8432a476666bbc13LLU);
-	}
-
-	*hash1 = ret1;
-	*hash2 = ret2;
+	hash(key, strlen((const char *)key), 0x94122f33U, r_hash);
 }
 
 bool
@@ -574,41 +543,16 @@ ckh_string_keycomp(const void *k1, const void *k2)
 }
 
 void
-ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1,
-    size_t *hash2)
+ckh_pointer_hash(const void *key, size_t r_hash[2])
 {
-	size_t ret1, ret2;
-	uint64_t h;
 	union {
 		const void	*v;
-		uint64_t	i;
+		size_t		i;
 	} u;
 
-	assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64));
-	assert(hash1 != NULL);
-	assert(hash2 != NULL);
-
 	assert(sizeof(u.v) == sizeof(u.i));
-#if (LG_SIZEOF_PTR != LG_SIZEOF_INT)
-	u.i = 0;
-#endif
 	u.v = key;
-	h = hash(&u.i, sizeof(u.i), 0xd983396e68886082LLU);
-	if (minbits <= 32) {
-		/*
-		 * Avoid doing multiple hashes, since a single hash provides
-		 * enough bits.
-		 */
-		ret1 = h & ZU(0xffffffffU);
-		ret2 = h >> 32;
-	} else {
-		assert(SIZEOF_PTR == 8);
-		ret1 = h;
-		ret2 = hash(&u.i, sizeof(u.i), 0x5e2be9aff8709a5dLLU);
-	}
-
-	*hash1 = ret1;
-	*hash2 = ret2;
+	hash(&u.i, sizeof(u.i), 0xd983396eU, r_hash);
 }
 
 bool

src/extent.c

@@ -3,7 +3,6 @@
 
 /******************************************************************************/
 
-#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS))
 static inline int
 extent_szad_comp(extent_node_t *a, extent_node_t *b)
 {
@@ -25,7 +24,6 @@ extent_szad_comp(extent_node_t *a, extent_node_t *b)
 /* Generate red-black tree functions. */
 rb_gen(, extent_tree_szad_, extent_tree_t, extent_node_t, link_szad,
     extent_szad_comp)
-#endif
 
 static inline int
 extent_ad_comp(extent_node_t *a, extent_node_t *b)

src/huge.c

@@ -4,11 +4,9 @@
 /******************************************************************************/
 /* Data. */
 
-#ifdef JEMALLOC_STATS
 uint64_t	huge_nmalloc;
 uint64_t	huge_ndalloc;
 size_t		huge_allocated;
-#endif
 
 malloc_mutex_t	huge_mtx;
 
@@ -19,10 +17,18 @@ static extent_tree_t	huge;
 
 void *
 huge_malloc(size_t size, bool zero)
+{
+
+	return (huge_palloc(size, chunksize, zero));
+}
+
+void *
+huge_palloc(size_t size, size_t alignment, bool zero)
 {
 	void *ret;
 	size_t csize;
 	extent_node_t *node;
+	bool is_zeroed;
 
 	/* Allocate one or more contiguous chunks for this request. */
 
@@ -37,7 +43,13 @@ huge_malloc(size_t size, bool zero)
 	if (node == NULL)
 		return (NULL);
 
-	ret = chunk_alloc(csize, false, &zero);
+	/*
+	 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
+	 * it is possible to make correct junk/zero fill decisions below.
+	 */
+	is_zeroed = zero;
+	ret = chunk_alloc(csize, alignment, false, &is_zeroed,
+	    chunk_dss_prec_get());
 	if (ret == NULL) {
 		base_node_dealloc(node);
 		return (NULL);
@@ -49,111 +61,24 @@ huge_malloc(size_t size, bool zero)
 
 	malloc_mutex_lock(&huge_mtx);
 	extent_tree_ad_insert(&huge, node);
-#ifdef JEMALLOC_STATS
-	stats_cactive_add(csize);
-	huge_nmalloc++;
-	huge_allocated += csize;
-#endif
+	if (config_stats) {
+		stats_cactive_add(csize);
+		huge_nmalloc++;
+		huge_allocated += csize;
+	}
 	malloc_mutex_unlock(&huge_mtx);
 
-#ifdef JEMALLOC_FILL
-	if (zero == false) {
+	if (config_fill && zero == false) {
 		if (opt_junk)
 			memset(ret, 0xa5, csize);
-		else if (opt_zero)
+		else if (opt_zero && is_zeroed == false)
 			memset(ret, 0, csize);
 	}
-#endif
 
 	return (ret);
 }
 
-/* Only handles large allocations that require more than chunk alignment. */
-void *
-huge_palloc(size_t size, size_t alignment, bool zero)
-{
-	void *ret;
-	size_t alloc_size, chunk_size, offset;
-	extent_node_t *node;
-
-	/*
-	 * This allocation requires alignment that is even larger than chunk
-	 * alignment.  This means that huge_malloc() isn't good enough.
-	 *
-	 * Allocate almost twice as many chunks as are demanded by the size or
-	 * alignment, in order to assure the alignment can be achieved, then
-	 * unmap leading and trailing chunks.
-	 */
-	assert(alignment > chunksize);
-
-	chunk_size = CHUNK_CEILING(size);
-
-	if (size >= alignment)
-		alloc_size = chunk_size + alignment - chunksize;
-	else
-		alloc_size = (alignment << 1) - chunksize;
-
-	/* Allocate an extent node with which to track the chunk. */
-	node = base_node_alloc();
-	if (node == NULL)
-		return (NULL);
-
-	ret = chunk_alloc(alloc_size, false, &zero);
-	if (ret == NULL) {
-		base_node_dealloc(node);
-		return (NULL);
-	}
-
-	offset = (uintptr_t)ret & (alignment - 1);
-	assert((offset & chunksize_mask) == 0);
-	assert(offset < alloc_size);
-	if (offset == 0) {
-		/* Trim trailing space. */
-		chunk_dealloc((void *)((uintptr_t)ret + chunk_size), alloc_size
-		    - chunk_size);
-	} else {
-		size_t trailsize;
-
-		/* Trim leading space. */
-		chunk_dealloc(ret, alignment - offset);
-
-		ret = (void *)((uintptr_t)ret + (alignment - offset));
-
-		trailsize = alloc_size - (alignment - offset) - chunk_size;
-		if (trailsize != 0) {
-		    /* Trim trailing space. */
-		    assert(trailsize < alloc_size);
-		    chunk_dealloc((void *)((uintptr_t)ret + chunk_size),
-			trailsize);
-		}
-	}
-
-	/* Insert node into huge. */
-	node->addr = ret;
-	node->size = chunk_size;
-
-	malloc_mutex_lock(&huge_mtx);
-	extent_tree_ad_insert(&huge, node);
-#ifdef JEMALLOC_STATS
-	stats_cactive_add(chunk_size);
-	huge_nmalloc++;
-	huge_allocated += chunk_size;
-#endif
-	malloc_mutex_unlock(&huge_mtx);
-
-#ifdef JEMALLOC_FILL
-	if (zero == false) {
-		if (opt_junk)
-			memset(ret, 0xa5, chunk_size);
-		else if (opt_zero)
-			memset(ret, 0, chunk_size);
-	}
-#endif
-
-	return (ret);
-}
-
-void *
+bool
 huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
 {
 
@@ -164,30 +89,23 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
 	    && CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size)
 	    && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) {
 		assert(CHUNK_CEILING(oldsize) == oldsize);
-#ifdef JEMALLOC_FILL
-		if (opt_junk && size < oldsize) {
-			memset((void *)((uintptr_t)ptr + size), 0x5a,
-			    oldsize - size);
-		}
-#endif
-		return (ptr);
+		return (false);
 	}
 
 	/* Reallocation would require a move. */
-	return (NULL);
+	return (true);
 }
 
 void *
 huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
-    size_t alignment, bool zero)
+    size_t alignment, bool zero, bool try_tcache_dalloc)
 {
 	void *ret;
 	size_t copysize;
 
 	/* Try to avoid moving the allocation. */
-	ret = huge_ralloc_no_move(ptr, oldsize, size, extra);
-	if (ret != NULL)
-		return (ret);
+	if (huge_ralloc_no_move(ptr, oldsize, size, extra) == false)
+		return (ptr);
 
 	/*
 	 * size and oldsize are different enough that we need to use a
@@ -218,22 +136,22 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
 	 */
 	copysize = (size < oldsize) ? size : oldsize;
 
+#ifdef JEMALLOC_MREMAP
 	/*
 	 * Use mremap(2) if this is a huge-->huge reallocation, and neither the
-	 * source nor the destination are in swap or dss.
+	 * source nor the destination are in dss.
 	 */
-#ifdef JEMALLOC_MREMAP_FIXED
-	if (oldsize >= chunksize
-#  ifdef JEMALLOC_SWAP
-	    && (swap_enabled == false || (chunk_in_swap(ptr) == false &&
-	    chunk_in_swap(ret) == false))
-#  endif
-#  ifdef JEMALLOC_DSS
-	    && chunk_in_dss(ptr) == false && chunk_in_dss(ret) == false
-#  endif
-	    ) {
+	if (oldsize >= chunksize && (config_dss == false || (chunk_in_dss(ptr)
+	    == false && chunk_in_dss(ret) == false))) {
 		size_t newsize = huge_salloc(ret);
 
+		/*
+		 * Remove ptr from the tree of huge allocations before
+		 * performing the remap operation, in order to avoid the
+		 * possibility of another thread acquiring that mapping before
+		 * this one removes it from the tree.
+		 */
+		huge_dalloc(ptr, false);
 		if (mremap(ptr, oldsize, newsize, MREMAP_MAYMOVE|MREMAP_FIXED,
 		    ret) == MAP_FAILED) {
 			/*
@@ -246,25 +164,46 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
 			 */
 			char buf[BUFERROR_BUF];
 
-			buferror(errno, buf, sizeof(buf));
-			malloc_write("<jemalloc>: Error in mremap(): ");
-			malloc_write(buf);
-			malloc_write("\n");
+			buferror(get_errno(), buf, sizeof(buf));
+			malloc_printf("<jemalloc>: Error in mremap(): %s\n",
+			    buf);
 			if (opt_abort)
 				abort();
 			memcpy(ret, ptr, copysize);
-			idalloc(ptr);
-		} else
-			huge_dalloc(ptr, false);
+			chunk_dealloc_mmap(ptr, oldsize);
+		}
 	} else
 #endif
 	{
 		memcpy(ret, ptr, copysize);
-		idalloc(ptr);
+		iqalloct(ptr, try_tcache_dalloc);
 	}
 	return (ret);
 }
 
+#ifdef JEMALLOC_JET
+#undef huge_dalloc_junk
+#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
+#endif
+static void
+huge_dalloc_junk(void *ptr, size_t usize)
+{
+
+	if (config_fill && config_dss && opt_junk) {
+		/*
+		 * Only bother junk filling if the chunk isn't about to be
+		 * unmapped.
+		 */
+		if (config_munmap == false || (config_dss && chunk_in_dss(ptr)))
+			memset(ptr, 0x5a, usize);
+	}
+}
+#ifdef JEMALLOC_JET
+#undef huge_dalloc_junk
+#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
+huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
+#endif
+
 void
 huge_dalloc(void *ptr, bool unmap)
 {
@@ -279,24 +218,18 @@ huge_dalloc(void *ptr, bool unmap)
 	assert(node->addr == ptr);
 	extent_tree_ad_remove(&huge, node);
 
-#ifdef JEMALLOC_STATS
-	stats_cactive_sub(node->size);
-	huge_ndalloc++;
-	huge_allocated -= node->size;
-#endif
+	if (config_stats) {
+		stats_cactive_sub(node->size);
+		huge_ndalloc++;
+		huge_allocated -= node->size;
+	}
 
 	malloc_mutex_unlock(&huge_mtx);
 
-	if (unmap) {
-	/* Unmap chunk. */
-#ifdef JEMALLOC_FILL
-#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS))
-		if (opt_junk)
-			memset(node->addr, 0x5a, node->size);
-#endif
-#endif
-		chunk_dealloc(node->addr, node->size);
-	}
+	if (unmap)
+		huge_dalloc_junk(node->addr, node->size);
+
+	chunk_dealloc(node->addr, node->size, unmap);
 
 	base_node_dealloc(node);
 }
@@ -321,7 +254,6 @@ huge_salloc(const void *ptr)
 	return (ret);
 }
 
-#ifdef JEMALLOC_PROF
 prof_ctx_t *
 huge_prof_ctx_get(const void *ptr)
 {
@@ -358,7 +290,6 @@ huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
 
 	malloc_mutex_unlock(&huge_mtx);
 }
-#endif
 
 bool
 huge_boot(void)
@@ -369,11 +300,32 @@ huge_boot(void)
 		return (true);
 	extent_tree_ad_new(&huge);
 
-#ifdef JEMALLOC_STATS
-	huge_nmalloc = 0;
-	huge_ndalloc = 0;
-	huge_allocated = 0;
-#endif
+	if (config_stats) {
+		huge_nmalloc = 0;
+		huge_ndalloc = 0;
+		huge_allocated = 0;
+	}
 
 	return (false);
 }
+
+void
+huge_prefork(void)
+{
+
+	malloc_mutex_prefork(&huge_mtx);
+}
+
+void
+huge_postfork_parent(void)
+{
+
+	malloc_mutex_postfork_parent(&huge_mtx);
+}
+
+void
+huge_postfork_child(void)
+{
+
+	malloc_mutex_postfork_child(&huge_mtx);
+}

src/mutex.c

@@ -1,14 +1,26 @@
 #define	JEMALLOC_MUTEX_C_
 #include "jemalloc/internal/jemalloc_internal.h"
 
+#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
+#include <dlfcn.h>
+#endif
+
+#ifndef _CRT_SPINCOUNT
+#define	_CRT_SPINCOUNT 4000
+#endif
+
 /******************************************************************************/
 /* Data. */
 
 #ifdef JEMALLOC_LAZY_LOCK
 bool isthreaded = false;
 #endif
+#ifdef JEMALLOC_MUTEX_INIT_CB
+static bool		postpone_init = true;
+static malloc_mutex_t	*postponed_mutexes = NULL;
+#endif
 
-#ifdef JEMALLOC_LAZY_LOCK
+#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
 static void	pthread_create_once(void);
 #endif
 
@@ -18,7 +30,7 @@ static void	pthread_create_once(void);
  * process goes multi-threaded.
  */
 
-#ifdef JEMALLOC_LAZY_LOCK
+#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
 static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *,
     void *(*)(void *), void *__restrict);
 
@@ -36,8 +48,7 @@ pthread_create_once(void)
 	isthreaded = true;
 }
 
-JEMALLOC_ATTR(visibility("default"))
-int
+JEMALLOC_EXPORT int
 pthread_create(pthread_t *__restrict thread,
     const pthread_attr_t *__restrict attr, void *(*start_routine)(void *),
     void *__restrict arg)
@@ -52,39 +63,87 @@ pthread_create(pthread_t *__restrict thread,
 
 /******************************************************************************/
 
+#ifdef JEMALLOC_MUTEX_INIT_CB
+JEMALLOC_EXPORT int	_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
+    void *(calloc_cb)(size_t, size_t));
+#endif
+
 bool
 malloc_mutex_init(malloc_mutex_t *mutex)
 {
-#ifdef JEMALLOC_OSSPIN
-	*mutex = 0;
+
+#ifdef _WIN32
+	if (!InitializeCriticalSectionAndSpinCount(&mutex->lock,
+	    _CRT_SPINCOUNT))
+		return (true);
+#elif (defined(JEMALLOC_OSSPIN))
+	mutex->lock = 0;
+#elif (defined(JEMALLOC_MUTEX_INIT_CB))
+	if (postpone_init) {
+		mutex->postponed_next = postponed_mutexes;
+		postponed_mutexes = mutex;
+	} else {
+		if (_pthread_mutex_init_calloc_cb(&mutex->lock, base_calloc) !=
+		    0)
+			return (true);
+	}
 #else
 	pthread_mutexattr_t attr;
 
 	if (pthread_mutexattr_init(&attr) != 0)
 		return (true);
-#ifdef PTHREAD_MUTEX_ADAPTIVE_NP
-	pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
-#else
-	pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT);
-#endif
-	if (pthread_mutex_init(mutex, &attr) != 0) {
+	pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE);
+	if (pthread_mutex_init(&mutex->lock, &attr) != 0) {
 		pthread_mutexattr_destroy(&attr);
 		return (true);
 	}
 	pthread_mutexattr_destroy(&attr);
-
 #endif
 	return (false);
 }
 
 void
-malloc_mutex_destroy(malloc_mutex_t *mutex)
+malloc_mutex_prefork(malloc_mutex_t *mutex)
 {
 
-#ifndef JEMALLOC_OSSPIN
-	if (pthread_mutex_destroy(mutex) != 0) {
-		malloc_write("<jemalloc>: Error in pthread_mutex_destroy()\n");
-		abort();
+	malloc_mutex_lock(mutex);
+}
+
+void
+malloc_mutex_postfork_parent(malloc_mutex_t *mutex)
+{
+
+	malloc_mutex_unlock(mutex);
+}
+
+void
+malloc_mutex_postfork_child(malloc_mutex_t *mutex)
+{
+
+#ifdef JEMALLOC_MUTEX_INIT_CB
+	malloc_mutex_unlock(mutex);
+#else
+	if (malloc_mutex_init(mutex)) {
+		malloc_printf("<jemalloc>: Error re-initializing mutex in "
+		    "child\n");
+		if (opt_abort)
+			abort();
 	}
 #endif
 }
+
+bool
+mutex_boot(void)
+{
+
+#ifdef JEMALLOC_MUTEX_INIT_CB
+	postpone_init = false;
+	while (postponed_mutexes != NULL) {
+		if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock,
+		    base_calloc) != 0)
+			return (true);
+		postponed_mutexes = postponed_mutexes->postponed_next;
+	}
+#endif
+	return (false);
+}

src/quarantine.c

@@ -0,0 +1,199 @@
+#define	JEMALLOC_QUARANTINE_C_
+#include "jemalloc/internal/jemalloc_internal.h"
+
+/*
+ * quarantine pointers close to NULL are used to encode state information that
+ * is used for cleaning up during thread shutdown.
+ */
+#define	QUARANTINE_STATE_REINCARNATED	((quarantine_t *)(uintptr_t)1)
+#define	QUARANTINE_STATE_PURGATORY	((quarantine_t *)(uintptr_t)2)
+#define	QUARANTINE_STATE_MAX		QUARANTINE_STATE_PURGATORY
+
+/******************************************************************************/
+/* Data. */
+
+malloc_tsd_data(, quarantine, quarantine_t *, NULL)
+
+/******************************************************************************/
+/* Function prototypes for non-inline static functions. */
+
+static quarantine_t	*quarantine_grow(quarantine_t *quarantine);
+static void	quarantine_drain_one(quarantine_t *quarantine);
+static void	quarantine_drain(quarantine_t *quarantine, size_t upper_bound);
+
+/******************************************************************************/
+
+quarantine_t *
+quarantine_init(size_t lg_maxobjs)
+{
+	quarantine_t *quarantine;
+
+	quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) +
+	    ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)));
+	if (quarantine == NULL)
+		return (NULL);
+	quarantine->curbytes = 0;
+	quarantine->curobjs = 0;
+	quarantine->first = 0;
+	quarantine->lg_maxobjs = lg_maxobjs;
+
+	quarantine_tsd_set(&quarantine);
+
+	return (quarantine);
+}
+
+static quarantine_t *
+quarantine_grow(quarantine_t *quarantine)
+{
+	quarantine_t *ret;
+
+	ret = quarantine_init(quarantine->lg_maxobjs + 1);
+	if (ret == NULL) {
+		quarantine_drain_one(quarantine);
+		return (quarantine);
+	}
+
+	ret->curbytes = quarantine->curbytes;
+	ret->curobjs = quarantine->curobjs;
+	if (quarantine->first + quarantine->curobjs <= (ZU(1) <<
+	    quarantine->lg_maxobjs)) {
+		/* objs ring buffer data are contiguous. */
+		memcpy(ret->objs, &quarantine->objs[quarantine->first],
+		    quarantine->curobjs * sizeof(quarantine_obj_t));
+	} else {
+		/* objs ring buffer data wrap around. */
+		size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) -
+		    quarantine->first;
+		size_t ncopy_b = quarantine->curobjs - ncopy_a;
+
+		memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a
+		    * sizeof(quarantine_obj_t));
+		memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
+		    sizeof(quarantine_obj_t));
+	}
+	idalloc(quarantine);
+
+	return (ret);
+}
+
+static void
+quarantine_drain_one(quarantine_t *quarantine)
+{
+	quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
+	assert(obj->usize == isalloc(obj->ptr, config_prof));
+	idalloc(obj->ptr);
+	quarantine->curbytes -= obj->usize;
+	quarantine->curobjs--;
+	quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
+	    quarantine->lg_maxobjs) - 1);
+}
+
+static void
+quarantine_drain(quarantine_t *quarantine, size_t upper_bound)
+{
+
+	while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
+		quarantine_drain_one(quarantine);
+}
+
+void
+quarantine(void *ptr)
+{
+	quarantine_t *quarantine;
+	size_t usize = isalloc(ptr, config_prof);
+
+	cassert(config_fill);
+	assert(opt_quarantine);
+
+	quarantine = *quarantine_tsd_get();
+	if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) {
+		if (quarantine == QUARANTINE_STATE_PURGATORY) {
+			/*
+			 * Make a note that quarantine() was called after
+			 * quarantine_cleanup() was called.
+			 */
+			quarantine = QUARANTINE_STATE_REINCARNATED;
+			quarantine_tsd_set(&quarantine);
+		}
+		idalloc(ptr);
+		return;
+	}
+	/*
+	 * Drain one or more objects if the quarantine size limit would be
+	 * exceeded by appending ptr.
+	 */
+	if (quarantine->curbytes + usize > opt_quarantine) {
+		size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
+		    - usize : 0;
+		quarantine_drain(quarantine, upper_bound);
+	}
+	/* Grow the quarantine ring buffer if it's full. */
+	if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
+		quarantine = quarantine_grow(quarantine);
+	/* quarantine_grow() must free a slot if it fails to grow. */
+	assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
+	/* Append ptr if its size doesn't exceed the quarantine size. */
+	if (quarantine->curbytes + usize <= opt_quarantine) {
+		size_t offset = (quarantine->first + quarantine->curobjs) &
+		    ((ZU(1) << quarantine->lg_maxobjs) - 1);
+		quarantine_obj_t *obj = &quarantine->objs[offset];
+		obj->ptr = ptr;
+		obj->usize = usize;
+		quarantine->curbytes += usize;
+		quarantine->curobjs++;
+		if (config_fill && opt_junk) {
+			/*
+			 * Only do redzone validation if Valgrind isn't in
+			 * operation.
+			 */
+			if ((config_valgrind == false || opt_valgrind == false)
+			    && usize <= SMALL_MAXCLASS)
+				arena_quarantine_junk_small(ptr, usize);
+			else
+				memset(ptr, 0x5a, usize);
+		}
+	} else {
+		assert(quarantine->curbytes == 0);
+		idalloc(ptr);
+	}
+}
+
+void
+quarantine_cleanup(void *arg)
+{
+	quarantine_t *quarantine = *(quarantine_t **)arg;
+
+	if (quarantine == QUARANTINE_STATE_REINCARNATED) {
+		/*
+		 * Another destructor deallocated memory after this destructor
+		 * was called.  Reset quarantine to QUARANTINE_STATE_PURGATORY
+		 * in order to receive another callback.
+		 */
+		quarantine = QUARANTINE_STATE_PURGATORY;
+		quarantine_tsd_set(&quarantine);
+	} else if (quarantine == QUARANTINE_STATE_PURGATORY) {
+		/*
+		 * The previous time this destructor was called, we set the key
+		 * to QUARANTINE_STATE_PURGATORY so that other destructors
+		 * wouldn't cause re-creation of the quarantine.  This time, do
+		 * nothing, so that the destructor will not be called again.
+		 */
+	} else if (quarantine != NULL) {
+		quarantine_drain(quarantine, 0);
+		idalloc(quarantine);
+		quarantine = QUARANTINE_STATE_PURGATORY;
+		quarantine_tsd_set(&quarantine);
+	}
+}
+
+bool
+quarantine_boot(void)
+{
+
+	cassert(config_fill);
+
+	if (quarantine_tsd_boot())
+		return (true);
+
+	return (false);
+}

src/rtree.c

@@ -2,45 +2,104 @@
 #include "jemalloc/internal/jemalloc_internal.h"
 
 rtree_t *
-rtree_new(unsigned bits)
+rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc)
 {
 	rtree_t *ret;
-	unsigned bits_per_level, height, i;
+	unsigned bits_per_level, bits_in_leaf, height, i;
+
+	assert(bits > 0 && bits <= (sizeof(uintptr_t) << 3));
 
 	bits_per_level = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void *)))) - 1;
-	height = bits / bits_per_level;
-	if (height * bits_per_level != bits)
-		height++;
-	assert(height * bits_per_level >= bits);
+	bits_in_leaf = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(uint8_t)))) - 1;
+	if (bits > bits_in_leaf) {
+		height = 1 + (bits - bits_in_leaf) / bits_per_level;
+		if ((height-1) * bits_per_level + bits_in_leaf != bits)
+			height++;
+	} else {
+		height = 1;
+	}
+	assert((height-1) * bits_per_level + bits_in_leaf >= bits);
 
-	ret = (rtree_t*)base_alloc(offsetof(rtree_t, level2bits) +
+	ret = (rtree_t*)alloc(offsetof(rtree_t, level2bits) +
 	    (sizeof(unsigned) * height));
 	if (ret == NULL)
 		return (NULL);
 	memset(ret, 0, offsetof(rtree_t, level2bits) + (sizeof(unsigned) *
 	    height));
 
+	ret->alloc = alloc;
+	ret->dalloc = dalloc;
 	if (malloc_mutex_init(&ret->mutex)) {
-		/* Leak the rtree. */
+		if (dalloc != NULL)
+			dalloc(ret);
 		return (NULL);
 	}
 	ret->height = height;
-	if (bits_per_level * height > bits)
-		ret->level2bits[0] = bits % bits_per_level;
-	else
-		ret->level2bits[0] = bits_per_level;
-	for (i = 1; i < height; i++)
-		ret->level2bits[i] = bits_per_level;
+	if (height > 1) {
+		if ((height-1) * bits_per_level + bits_in_leaf > bits) {
+			ret->level2bits[0] = (bits - bits_in_leaf) %
+			    bits_per_level;
+		} else
+			ret->level2bits[0] = bits_per_level;
+		for (i = 1; i < height-1; i++)
+			ret->level2bits[i] = bits_per_level;
+		ret->level2bits[height-1] = bits_in_leaf;
+	} else
+		ret->level2bits[0] = bits;
 
-	ret->root = (void**)base_alloc(sizeof(void *) << ret->level2bits[0]);
+	ret->root = (void**)alloc(sizeof(void *) << ret->level2bits[0]);
 	if (ret->root == NULL) {
-		/*
-		 * We leak the rtree here, since there's no generic base
-		 * deallocation.
-		 */
+		if (dalloc != NULL)
+			dalloc(ret);
 		return (NULL);
 	}
 	memset(ret->root, 0, sizeof(void *) << ret->level2bits[0]);
 
 	return (ret);
 }
+
+static void
+rtree_delete_subtree(rtree_t *rtree, void **node, unsigned level)
+{
+
+	if (level < rtree->height - 1) {
+		size_t nchildren, i;
+
+		nchildren = ZU(1) << rtree->level2bits[level];
+		for (i = 0; i < nchildren; i++) {
+			void **child = (void **)node[i];
+			if (child != NULL)
+				rtree_delete_subtree(rtree, child, level + 1);
+		}
+	}
+	rtree->dalloc(node);
+}
+
+void
+rtree_delete(rtree_t *rtree)
+{
+
+	rtree_delete_subtree(rtree, rtree->root, 0);
+	rtree->dalloc(rtree);
+}
+
+void
+rtree_prefork(rtree_t *rtree)
+{
+
+	malloc_mutex_prefork(&rtree->mutex);
+}
+
+void
+rtree_postfork_parent(rtree_t *rtree)
+{
+
+	malloc_mutex_postfork_parent(&rtree->mutex);
+}
+
+void
+rtree_postfork_child(rtree_t *rtree)
+{
+
+	malloc_mutex_postfork_child(&rtree->mutex);
+}

src/stats.c

@@ -39,140 +39,40 @@
 
 bool	opt_stats_print = false;
 
-#ifdef JEMALLOC_STATS
 size_t	stats_cactive = 0;
-#endif
 
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
 
-#ifdef JEMALLOC_STATS
-static void	malloc_vcprintf(void (*write_cb)(void *, const char *),
-    void *cbopaque, const char *format, va_list ap);
 static void	stats_arena_bins_print(void (*write_cb)(void *, const char *),
     void *cbopaque, unsigned i);
 static void	stats_arena_lruns_print(void (*write_cb)(void *, const char *),
     void *cbopaque, unsigned i);
 static void	stats_arena_print(void (*write_cb)(void *, const char *),
-    void *cbopaque, unsigned i);
-#endif
+    void *cbopaque, unsigned i, bool bins, bool large);
 
 /******************************************************************************/
 
-/*
- * We don't want to depend on vsnprintf() for production builds, since that can
- * cause unnecessary bloat for static binaries.  u2s() provides minimal integer
- * printing functionality, so that malloc_printf() use can be limited to
- * JEMALLOC_STATS code.
- */
-char *
-u2s(uint64_t x, unsigned base, char *s)
-{
-	unsigned i;
-
-	i = UMAX2S_BUFSIZE - 1;
-	s[i] = '\0';
-	switch (base) {
-	case 10:
-		do {
-			i--;
-			s[i] = "0123456789"[x % (uint64_t)10];
-			x /= (uint64_t)10;
-		} while (x > 0);
-		break;
-	case 16:
-		do {
-			i--;
-			s[i] = "0123456789abcdef"[x & 0xf];
-			x >>= 4;
-		} while (x > 0);
-		break;
-	default:
-		do {
-			i--;
-			s[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[x %
-			    (uint64_t)base];
-			x /= (uint64_t)base;
-		} while (x > 0);
-	}
-
-	return (&s[i]);
-}
-
-#ifdef JEMALLOC_STATS
-static void
-malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
-    const char *format, va_list ap)
-{
-	char buf[4096];
-
-	if (write_cb == NULL) {
-		/*
-		 * The caller did not provide an alternate write_cb callback
-		 * function, so use the default one.  malloc_write() is an
-		 * inline function, so use malloc_message() directly here.
-		 */
-		write_cb = JEMALLOC_P(malloc_message);
-		cbopaque = NULL;
-	}
-
-	vsnprintf(buf, sizeof(buf), format, ap);
-	write_cb(cbopaque, buf);
-}
-
-/*
- * Print to a callback function in such a way as to (hopefully) avoid memory
- * allocation.
- */
-JEMALLOC_ATTR(format(printf, 3, 4))
-void
-malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque,
-    const char *format, ...)
-{
-	va_list ap;
-
-	va_start(ap, format);
-	malloc_vcprintf(write_cb, cbopaque, format, ap);
-	va_end(ap);
-}
-
-/*
- * Print to stderr in such a way as to (hopefully) avoid memory allocation.
- */
-JEMALLOC_ATTR(format(printf, 1, 2))
-void
-malloc_printf(const char *format, ...)
-{
-	va_list ap;
-
-	va_start(ap, format);
-	malloc_vcprintf(NULL, NULL, format, ap);
-	va_end(ap);
-}
-#endif
-
-#ifdef JEMALLOC_STATS
 static void
 stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
     unsigned i)
 {
-	size_t pagesize;
+	size_t page;
 	bool config_tcache;
 	unsigned nbins, j, gap_start;
 
-	CTL_GET("arenas.pagesize", &pagesize, size_t);
+	CTL_GET("arenas.page", &page, size_t);
 
 	CTL_GET("config.tcache", &config_tcache, bool);
 	if (config_tcache) {
 		malloc_cprintf(write_cb, cbopaque,
-		    "bins:     bin    size regs pgs    allocated      nmalloc"
+		    "bins:     bin  size regs pgs    allocated      nmalloc"
 		    "      ndalloc    nrequests       nfills     nflushes"
-		    "      newruns       reruns      maxruns      curruns\n");
+		    "      newruns       reruns      curruns\n");
 	} else {
 		malloc_cprintf(write_cb, cbopaque,
-		    "bins:     bin    size regs pgs    allocated      nmalloc"
-		    "      ndalloc      newruns       reruns      maxruns"
-		    "      curruns\n");
+		    "bins:     bin  size regs pgs    allocated      nmalloc"
+		    "      ndalloc      newruns       reruns      curruns\n");
 	}
 	CTL_GET("arenas.nbins", &nbins, unsigned);
 	for (j = 0, gap_start = UINT_MAX; j < nbins; j++) {
@@ -183,12 +83,11 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
 			if (gap_start == UINT_MAX)
 				gap_start = j;
 		} else {
-			unsigned ntbins_, nqbins, ncbins, nsbins;
 			size_t reg_size, run_size, allocated;
 			uint32_t nregs;
 			uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes;
 			uint64_t reruns;
-			size_t highruns, curruns;
+			size_t curruns;
 
 			if (gap_start != UINT_MAX) {
 				if (j > gap_start + 1) {
@@ -203,10 +102,6 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
 				}
 				gap_start = UINT_MAX;
 			}
-			CTL_GET("arenas.ntbins", &ntbins_, unsigned);
-			CTL_GET("arenas.nqbins", &nqbins, unsigned);
-			CTL_GET("arenas.ncbins", &ncbins, unsigned);
-			CTL_GET("arenas.nsbins", &nsbins, unsigned);
 			CTL_J_GET("arenas.bin.0.size", &reg_size, size_t);
 			CTL_J_GET("arenas.bin.0.nregs", &nregs, uint32_t);
 			CTL_J_GET("arenas.bin.0.run_size", &run_size, size_t);
@@ -226,36 +121,25 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
 			}
 			CTL_IJ_GET("stats.arenas.0.bins.0.nreruns", &reruns,
 			    uint64_t);
-			CTL_IJ_GET("stats.arenas.0.bins.0.highruns", &highruns,
-			    size_t);
 			CTL_IJ_GET("stats.arenas.0.bins.0.curruns", &curruns,
 			    size_t);
 			if (config_tcache) {
 				malloc_cprintf(write_cb, cbopaque,
-				    "%13u %1s %5zu %4u %3zu %12zu %12"PRIu64
+				    "%13u %5zu %4u %3zu %12zu %12"PRIu64
 				    " %12"PRIu64" %12"PRIu64" %12"PRIu64
 				    " %12"PRIu64" %12"PRIu64" %12"PRIu64
-				    " %12zu %12zu\n",
-				    j,
-				    j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
-				    "Q" : j < ntbins_ + nqbins + ncbins ? "C" :
-				    "S",
-				    reg_size, nregs, run_size / pagesize,
+				    " %12zu\n",
+				    j, reg_size, nregs, run_size / page,
 				    allocated, nmalloc, ndalloc, nrequests,
-				    nfills, nflushes, nruns, reruns, highruns,
-				    curruns);
+				    nfills, nflushes, nruns, reruns, curruns);
 			} else {
 				malloc_cprintf(write_cb, cbopaque,
-				    "%13u %1s %5zu %4u %3zu %12zu %12"PRIu64
+				    "%13u %5zu %4u %3zu %12zu %12"PRIu64
 				    " %12"PRIu64" %12"PRIu64" %12"PRIu64
-				    " %12zu %12zu\n",
-				    j,
-				    j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
-				    "Q" : j < ntbins_ + nqbins + ncbins ? "C" :
-				    "S",
-				    reg_size, nregs, run_size / pagesize,
+				    " %12zu\n",
+				    j, reg_size, nregs, run_size / page,
 				    allocated, nmalloc, ndalloc, nruns, reruns,
-				    highruns, curruns);
+				    curruns);
 			}
 		}
 	}
@@ -275,18 +159,18 @@ static void
 stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque,
     unsigned i)
 {
-	size_t pagesize, nlruns, j;
+	size_t page, nlruns, j;
 	ssize_t gap_start;
 
-	CTL_GET("arenas.pagesize", &pagesize, size_t);
+	CTL_GET("arenas.page", &page, size_t);
 
 	malloc_cprintf(write_cb, cbopaque,
 	    "large:   size pages      nmalloc      ndalloc    nrequests"
-	    "      maxruns      curruns\n");
+	    "      curruns\n");
 	CTL_GET("arenas.nlruns", &nlruns, size_t);
 	for (j = 0, gap_start = -1; j < nlruns; j++) {
 		uint64_t nmalloc, ndalloc, nrequests;
-		size_t run_size, highruns, curruns;
+		size_t run_size, curruns;
 
 		CTL_IJ_GET("stats.arenas.0.lruns.0.nmalloc", &nmalloc,
 		    uint64_t);
@@ -299,8 +183,6 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque,
 				gap_start = j;
 		} else {
 			CTL_J_GET("arenas.lrun.0.size", &run_size, size_t);
-			CTL_IJ_GET("stats.arenas.0.lruns.0.highruns", &highruns,
-			    size_t);
 			CTL_IJ_GET("stats.arenas.0.lruns.0.curruns", &curruns,
 			    size_t);
 			if (gap_start != -1) {
@@ -310,9 +192,9 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque,
 			}
 			malloc_cprintf(write_cb, cbopaque,
 			    "%13zu %5zu %12"PRIu64" %12"PRIu64" %12"PRIu64
-			    " %12zu %12zu\n",
-			    run_size, run_size / pagesize, nmalloc, ndalloc,
-			    nrequests, highruns, curruns);
+			    " %12zu\n",
+			    run_size, run_size / page, nmalloc, ndalloc,
+			    nrequests, curruns);
 		}
 	}
 	if (gap_start != -1)
@@ -321,21 +203,25 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque,
 
 static void
 stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
-    unsigned i)
+    unsigned i, bool bins, bool large)
 {
 	unsigned nthreads;
-	size_t pagesize, pactive, pdirty, mapped;
+	const char *dss;
+	size_t page, pactive, pdirty, mapped;
 	uint64_t npurge, nmadvise, purged;
 	size_t small_allocated;
 	uint64_t small_nmalloc, small_ndalloc, small_nrequests;
 	size_t large_allocated;
 	uint64_t large_nmalloc, large_ndalloc, large_nrequests;
 
-	CTL_GET("arenas.pagesize", &pagesize, size_t);
+	CTL_GET("arenas.page", &page, size_t);
 
 	CTL_I_GET("stats.arenas.0.nthreads", &nthreads, unsigned);
 	malloc_cprintf(write_cb, cbopaque,
 	    "assigned threads: %u\n", nthreads);
+	CTL_I_GET("stats.arenas.0.dss", &dss, const char *);
+	malloc_cprintf(write_cb, cbopaque, "dss allocation precedence: %s\n",
+	    dss);
 	CTL_I_GET("stats.arenas.0.pactive", &pactive, size_t);
 	CTL_I_GET("stats.arenas.0.pdirty", &pdirty, size_t);
 	CTL_I_GET("stats.arenas.0.npurge", &npurge, uint64_t);
@@ -369,15 +255,15 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
 	    small_nmalloc + large_nmalloc,
 	    small_ndalloc + large_ndalloc,
 	    small_nrequests + large_nrequests);
-	malloc_cprintf(write_cb, cbopaque, "active:  %12zu\n",
-	    pactive * pagesize );
+	malloc_cprintf(write_cb, cbopaque, "active:  %12zu\n", pactive * page);
 	CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t);
 	malloc_cprintf(write_cb, cbopaque, "mapped:  %12zu\n", mapped);
 
-	stats_arena_bins_print(write_cb, cbopaque, i);
-	stats_arena_lruns_print(write_cb, cbopaque, i);
+	if (bins)
+		stats_arena_bins_print(write_cb, cbopaque, i);
+	if (large)
+		stats_arena_lruns_print(write_cb, cbopaque, i);
 }
-#endif
 
 void
 stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
@@ -386,7 +272,6 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 	int err;
 	uint64_t epoch;
 	size_t u64sz;
-	char s[UMAX2S_BUFSIZE];
 	bool general = true;
 	bool merged = true;
 	bool unmerged = true;
@@ -402,8 +287,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 	 * */
 	epoch = 1;
 	u64sz = sizeof(uint64_t);
-	err = JEMALLOC_P(mallctl)("epoch", &epoch, &u64sz, &epoch,
-	    sizeof(uint64_t));
+	err = je_mallctl("epoch", &epoch, &u64sz, &epoch, sizeof(uint64_t));
 	if (err != 0) {
 		if (err == EAGAIN) {
 			malloc_write("<jemalloc>: Memory allocation failure in "
@@ -415,42 +299,33 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 		abort();
 	}
 
-	if (write_cb == NULL) {
-		/*
-		 * The caller did not provide an alternate write_cb callback
-		 * function, so use the default one.  malloc_write() is an
-		 * inline function, so use malloc_message() directly here.
-		 */
-		write_cb = JEMALLOC_P(malloc_message);
-		cbopaque = NULL;
-	}
-
 	if (opts != NULL) {
 		unsigned i;
 
 		for (i = 0; opts[i] != '\0'; i++) {
 			switch (opts[i]) {
-				case 'g':
-					general = false;
-					break;
-				case 'm':
-					merged = false;
-					break;
-				case 'a':
-					unmerged = false;
-					break;
-				case 'b':
-					bins = false;
-					break;
-				case 'l':
-					large = false;
-					break;
-				default:;
+			case 'g':
+				general = false;
+				break;
+			case 'm':
+				merged = false;
+				break;
+			case 'a':
+				unmerged = false;
+				break;
+			case 'b':
+				bins = false;
+				break;
+			case 'l':
+				large = false;
+				break;
+			default:;
 			}
 		}
 	}
 
-	write_cb(cbopaque, "___ Begin jemalloc statistics ___\n");
+	malloc_cprintf(write_cb, cbopaque,
+	    "___ Begin jemalloc statistics ___\n");
 	if (general) {
 		int err;
 		const char *cpv;
@@ -465,229 +340,127 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 		cpsz = sizeof(const char *);
 
 		CTL_GET("version", &cpv, const char *);
-		write_cb(cbopaque, "Version: ");
-		write_cb(cbopaque, cpv);
-		write_cb(cbopaque, "\n");
+		malloc_cprintf(write_cb, cbopaque, "Version: %s\n", cpv);
 		CTL_GET("config.debug", &bv, bool);
-		write_cb(cbopaque, "Assertions ");
-		write_cb(cbopaque, bv ? "enabled" : "disabled");
-		write_cb(cbopaque, "\n");
+		malloc_cprintf(write_cb, cbopaque, "Assertions %s\n",
+		    bv ? "enabled" : "disabled");
 
-#define OPT_WRITE_BOOL(n)						\
-		if ((err = JEMALLOC_P(mallctl)("opt."#n, &bv, &bsz,	\
-		    NULL, 0)) == 0) {					\
-			write_cb(cbopaque, "  opt."#n": ");		\
-			write_cb(cbopaque, bv ? "true" : "false");	\
-			write_cb(cbopaque, "\n");			\
+#define	OPT_WRITE_BOOL(n)						\
+		if ((err = je_mallctl("opt."#n, &bv, &bsz, NULL, 0))	\
+		    == 0) {						\
+			malloc_cprintf(write_cb, cbopaque,		\
+			    "  opt."#n": %s\n", bv ? "true" : "false");	\
 		}
-#define OPT_WRITE_SIZE_T(n)						\
-		if ((err = JEMALLOC_P(mallctl)("opt."#n, &sv, &ssz,	\
-		    NULL, 0)) == 0) {					\
-			write_cb(cbopaque, "  opt."#n": ");		\
-			write_cb(cbopaque, u2s(sv, 10, s));		\
-			write_cb(cbopaque, "\n");			\
+#define	OPT_WRITE_SIZE_T(n)						\
+		if ((err = je_mallctl("opt."#n, &sv, &ssz, NULL, 0))	\
+		    == 0) {						\
+			malloc_cprintf(write_cb, cbopaque,		\
+			"  opt."#n": %zu\n", sv);			\
 		}
-#define OPT_WRITE_SSIZE_T(n)						\
-		if ((err = JEMALLOC_P(mallctl)("opt."#n, &ssv, &sssz,	\
-		    NULL, 0)) == 0) {					\
-			if (ssv >= 0) {					\
-				write_cb(cbopaque, "  opt."#n": ");	\
-				write_cb(cbopaque, u2s(ssv, 10, s));	\
-			} else {					\
-				write_cb(cbopaque, "  opt."#n": -");	\
-				write_cb(cbopaque, u2s(-ssv, 10, s));	\
-			}						\
-			write_cb(cbopaque, "\n");			\
+#define	OPT_WRITE_SSIZE_T(n)						\
+		if ((err = je_mallctl("opt."#n, &ssv, &sssz, NULL, 0))	\
+		    == 0) {						\
+			malloc_cprintf(write_cb, cbopaque,		\
+			    "  opt."#n": %zd\n", ssv);			\
 		}
-#define OPT_WRITE_CHAR_P(n)						\
-		if ((err = JEMALLOC_P(mallctl)("opt."#n, &cpv, &cpsz,	\
-		    NULL, 0)) == 0) {					\
-			write_cb(cbopaque, "  opt."#n": \"");		\
-			write_cb(cbopaque, cpv);			\
-			write_cb(cbopaque, "\"\n");			\
+#define	OPT_WRITE_CHAR_P(n)						\
+		if ((err = je_mallctl("opt."#n, &cpv, &cpsz, NULL, 0))	\
+		    == 0) {						\
+			malloc_cprintf(write_cb, cbopaque,		\
+			    "  opt."#n": \"%s\"\n", cpv);		\
 		}
 
-		write_cb(cbopaque, "Run-time option settings:\n");
+		malloc_cprintf(write_cb, cbopaque,
+		    "Run-time option settings:\n");
 		OPT_WRITE_BOOL(abort)
-		OPT_WRITE_SIZE_T(lg_qspace_max)
-		OPT_WRITE_SIZE_T(lg_cspace_max)
 		OPT_WRITE_SIZE_T(lg_chunk)
+		OPT_WRITE_CHAR_P(dss)
 		OPT_WRITE_SIZE_T(narenas)
 		OPT_WRITE_SSIZE_T(lg_dirty_mult)
 		OPT_WRITE_BOOL(stats_print)
 		OPT_WRITE_BOOL(junk)
+		OPT_WRITE_SIZE_T(quarantine)
+		OPT_WRITE_BOOL(redzone)
 		OPT_WRITE_BOOL(zero)
-		OPT_WRITE_BOOL(sysv)
+		OPT_WRITE_BOOL(utrace)
+		OPT_WRITE_BOOL(valgrind)
 		OPT_WRITE_BOOL(xmalloc)
 		OPT_WRITE_BOOL(tcache)
-		OPT_WRITE_SSIZE_T(lg_tcache_gc_sweep)
 		OPT_WRITE_SSIZE_T(lg_tcache_max)
 		OPT_WRITE_BOOL(prof)
 		OPT_WRITE_CHAR_P(prof_prefix)
-		OPT_WRITE_SIZE_T(lg_prof_bt_max)
 		OPT_WRITE_BOOL(prof_active)
 		OPT_WRITE_SSIZE_T(lg_prof_sample)
 		OPT_WRITE_BOOL(prof_accum)
-		OPT_WRITE_SSIZE_T(lg_prof_tcmax)
 		OPT_WRITE_SSIZE_T(lg_prof_interval)
 		OPT_WRITE_BOOL(prof_gdump)
+		OPT_WRITE_BOOL(prof_final)
 		OPT_WRITE_BOOL(prof_leak)
-		OPT_WRITE_BOOL(overcommit)
 
 #undef OPT_WRITE_BOOL
 #undef OPT_WRITE_SIZE_T
 #undef OPT_WRITE_SSIZE_T
 #undef OPT_WRITE_CHAR_P
 
-		write_cb(cbopaque, "CPUs: ");
-		write_cb(cbopaque, u2s(ncpus, 10, s));
-		write_cb(cbopaque, "\n");
+		malloc_cprintf(write_cb, cbopaque, "CPUs: %u\n", ncpus);
 
 		CTL_GET("arenas.narenas", &uv, unsigned);
-		write_cb(cbopaque, "Max arenas: ");
-		write_cb(cbopaque, u2s(uv, 10, s));
-		write_cb(cbopaque, "\n");
+		malloc_cprintf(write_cb, cbopaque, "Arenas: %u\n", uv);
 
-		write_cb(cbopaque, "Pointer size: ");
-		write_cb(cbopaque, u2s(sizeof(void *), 10, s));
-		write_cb(cbopaque, "\n");
+		malloc_cprintf(write_cb, cbopaque, "Pointer size: %zu\n",
+		    sizeof(void *));
 
 		CTL_GET("arenas.quantum", &sv, size_t);
-		write_cb(cbopaque, "Quantum size: ");
-		write_cb(cbopaque, u2s(sv, 10, s));
-		write_cb(cbopaque, "\n");
+		malloc_cprintf(write_cb, cbopaque, "Quantum size: %zu\n", sv);
 
-		CTL_GET("arenas.cacheline", &sv, size_t);
-		write_cb(cbopaque, "Cacheline size (assumed): ");
-		write_cb(cbopaque, u2s(sv, 10, s));
-		write_cb(cbopaque, "\n");
-
-		CTL_GET("arenas.subpage", &sv, size_t);
-		write_cb(cbopaque, "Subpage spacing: ");
-		write_cb(cbopaque, u2s(sv, 10, s));
-		write_cb(cbopaque, "\n");
-
-		if ((err = JEMALLOC_P(mallctl)("arenas.tspace_min", &sv, &ssz,
-		    NULL, 0)) == 0) {
-			write_cb(cbopaque, "Tiny 2^n-spaced sizes: [");
-			write_cb(cbopaque, u2s(sv, 10, s));
-			write_cb(cbopaque, "..");
-
-			CTL_GET("arenas.tspace_max", &sv, size_t);
-			write_cb(cbopaque, u2s(sv, 10, s));
-			write_cb(cbopaque, "]\n");
-		}
-
-		CTL_GET("arenas.qspace_min", &sv, size_t);
-		write_cb(cbopaque, "Quantum-spaced sizes: [");
-		write_cb(cbopaque, u2s(sv, 10, s));
-		write_cb(cbopaque, "..");
-		CTL_GET("arenas.qspace_max", &sv, size_t);
-		write_cb(cbopaque, u2s(sv, 10, s));
-		write_cb(cbopaque, "]\n");
-
-		CTL_GET("arenas.cspace_min", &sv, size_t);
-		write_cb(cbopaque, "Cacheline-spaced sizes: [");
-		write_cb(cbopaque, u2s(sv, 10, s));
-		write_cb(cbopaque, "..");
-		CTL_GET("arenas.cspace_max", &sv, size_t);
-		write_cb(cbopaque, u2s(sv, 10, s));
-		write_cb(cbopaque, "]\n");
-
-		CTL_GET("arenas.sspace_min", &sv, size_t);
-		write_cb(cbopaque, "Subpage-spaced sizes: [");
-		write_cb(cbopaque, u2s(sv, 10, s));
-		write_cb(cbopaque, "..");
-		CTL_GET("arenas.sspace_max", &sv, size_t);
-		write_cb(cbopaque, u2s(sv, 10, s));
-		write_cb(cbopaque, "]\n");
+		CTL_GET("arenas.page", &sv, size_t);
+		malloc_cprintf(write_cb, cbopaque, "Page size: %zu\n", sv);
 
 		CTL_GET("opt.lg_dirty_mult", &ssv, ssize_t);
 		if (ssv >= 0) {
-			write_cb(cbopaque,
-			    "Min active:dirty page ratio per arena: ");
-			write_cb(cbopaque, u2s((1U << ssv), 10, s));
-			write_cb(cbopaque, ":1\n");
+			malloc_cprintf(write_cb, cbopaque,
+			    "Min active:dirty page ratio per arena: %u:1\n",
+			    (1U << ssv));
 		} else {
-			write_cb(cbopaque,
+			malloc_cprintf(write_cb, cbopaque,
 			    "Min active:dirty page ratio per arena: N/A\n");
 		}
-		if ((err = JEMALLOC_P(mallctl)("arenas.tcache_max", &sv,
-		    &ssz, NULL, 0)) == 0) {
-			write_cb(cbopaque,
-			    "Maximum thread-cached size class: ");
-			write_cb(cbopaque, u2s(sv, 10, s));
-			write_cb(cbopaque, "\n");
+		if ((err = je_mallctl("arenas.tcache_max", &sv, &ssz, NULL, 0))
+		    == 0) {
+			malloc_cprintf(write_cb, cbopaque,
+			    "Maximum thread-cached size class: %zu\n", sv);
 		}
-		if ((err = JEMALLOC_P(mallctl)("opt.lg_tcache_gc_sweep", &ssv,
-		    &ssz, NULL, 0)) == 0) {
-			size_t tcache_gc_sweep = (1U << ssv);
-			bool tcache_enabled;
-			CTL_GET("opt.tcache", &tcache_enabled, bool);
-			write_cb(cbopaque, "Thread cache GC sweep interval: ");
-			write_cb(cbopaque, tcache_enabled && ssv >= 0 ?
-			    u2s(tcache_gc_sweep, 10, s) : "N/A");
-			write_cb(cbopaque, "\n");
-		}
-		if ((err = JEMALLOC_P(mallctl)("opt.prof", &bv, &bsz, NULL, 0))
-		   == 0 && bv) {
-			CTL_GET("opt.lg_prof_bt_max", &sv, size_t);
-			write_cb(cbopaque, "Maximum profile backtrace depth: ");
-			write_cb(cbopaque, u2s((1U << sv), 10, s));
-			write_cb(cbopaque, "\n");
-
-			CTL_GET("opt.lg_prof_tcmax", &ssv, ssize_t);
-			write_cb(cbopaque,
-			    "Maximum per thread backtrace cache: ");
-			if (ssv >= 0) {
-				write_cb(cbopaque, u2s((1U << ssv), 10, s));
-				write_cb(cbopaque, " (2^");
-				write_cb(cbopaque, u2s(ssv, 10, s));
-				write_cb(cbopaque, ")\n");
-			} else
-				write_cb(cbopaque, "N/A\n");
-
+		if ((err = je_mallctl("opt.prof", &bv, &bsz, NULL, 0)) == 0 &&
+		    bv) {
 			CTL_GET("opt.lg_prof_sample", &sv, size_t);
-			write_cb(cbopaque, "Average profile sample interval: ");
-			write_cb(cbopaque, u2s((((uint64_t)1U) << sv), 10, s));
-			write_cb(cbopaque, " (2^");
-			write_cb(cbopaque, u2s(sv, 10, s));
-			write_cb(cbopaque, ")\n");
+			malloc_cprintf(write_cb, cbopaque,
+			    "Average profile sample interval: %"PRIu64
+			    " (2^%zu)\n", (((uint64_t)1U) << sv), sv);
 
 			CTL_GET("opt.lg_prof_interval", &ssv, ssize_t);
-			write_cb(cbopaque, "Average profile dump interval: ");
 			if (ssv >= 0) {
-				write_cb(cbopaque, u2s((((uint64_t)1U) << ssv),
-				    10, s));
-				write_cb(cbopaque, " (2^");
-				write_cb(cbopaque, u2s(ssv, 10, s));
-				write_cb(cbopaque, ")\n");
-			} else
-				write_cb(cbopaque, "N/A\n");
+				malloc_cprintf(write_cb, cbopaque,
+				    "Average profile dump interval: %"PRIu64
+				    " (2^%zd)\n",
+				    (((uint64_t)1U) << ssv), ssv);
+			} else {
+				malloc_cprintf(write_cb, cbopaque,
+				    "Average profile dump interval: N/A\n");
+			}
 		}
-		CTL_GET("arenas.chunksize", &sv, size_t);
-		write_cb(cbopaque, "Chunk size: ");
-		write_cb(cbopaque, u2s(sv, 10, s));
 		CTL_GET("opt.lg_chunk", &sv, size_t);
-		write_cb(cbopaque, " (2^");
-		write_cb(cbopaque, u2s(sv, 10, s));
-		write_cb(cbopaque, ")\n");
+		malloc_cprintf(write_cb, cbopaque, "Chunk size: %zu (2^%zu)\n",
+		    (ZU(1) << sv), sv);
 	}
 
-#ifdef JEMALLOC_STATS
-	{
-		int err;
-		size_t sszp, ssz;
+	if (config_stats) {
 		size_t *cactive;
 		size_t allocated, active, mapped;
-		size_t chunks_current, chunks_high, swap_avail;
+		size_t chunks_current, chunks_high;
 		uint64_t chunks_total;
 		size_t huge_allocated;
 		uint64_t huge_nmalloc, huge_ndalloc;
 
-		sszp = sizeof(size_t *);
-		ssz = sizeof(size_t);
-
 		CTL_GET("stats.cactive", &cactive, size_t *);
 		CTL_GET("stats.allocated", &allocated, size_t);
 		CTL_GET("stats.active", &active, size_t);
@@ -702,24 +475,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 		CTL_GET("stats.chunks.total", &chunks_total, uint64_t);
 		CTL_GET("stats.chunks.high", &chunks_high, size_t);
 		CTL_GET("stats.chunks.current", &chunks_current, size_t);
-		if ((err = JEMALLOC_P(mallctl)("swap.avail", &swap_avail, &ssz,
-		    NULL, 0)) == 0) {
-			size_t lg_chunk;
-
-			malloc_cprintf(write_cb, cbopaque, "chunks: nchunks   "
-			    "highchunks    curchunks   swap_avail\n");
-			CTL_GET("opt.lg_chunk", &lg_chunk, size_t);
-			malloc_cprintf(write_cb, cbopaque,
-			    "  %13"PRIu64"%13zu%13zu%13zu\n",
-			    chunks_total, chunks_high, chunks_current,
-			    swap_avail << lg_chunk);
-		} else {
-			malloc_cprintf(write_cb, cbopaque, "chunks: nchunks   "
-			    "highchunks    curchunks\n");
-			malloc_cprintf(write_cb, cbopaque,
-			    "  %13"PRIu64"%13zu%13zu\n",
-			    chunks_total, chunks_high, chunks_current);
-		}
+		malloc_cprintf(write_cb, cbopaque, "chunks: nchunks   "
+		    "highchunks    curchunks\n");
+		malloc_cprintf(write_cb, cbopaque,
+		    "  %13"PRIu64" %12zu %12zu\n",
+		    chunks_total, chunks_high, chunks_current);
 
 		/* Print huge stats. */
 		CTL_GET("stats.huge.nmalloc", &huge_nmalloc, uint64_t);
@@ -736,11 +496,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 
 			CTL_GET("arenas.narenas", &narenas, unsigned);
 			{
-				bool initialized[narenas];
+				VARIABLE_ARRAY(bool, initialized, narenas);
 				size_t isz;
 				unsigned i, ninitialized;
 
-				isz = sizeof(initialized);
+				isz = sizeof(bool) * narenas;
 				xmallctl("arenas.initialized", initialized,
 				    &isz, NULL, 0);
 				for (i = ninitialized = 0; i < narenas; i++) {
@@ -748,12 +508,12 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 						ninitialized++;
 				}
 
-				if (ninitialized > 1) {
+				if (ninitialized > 1 || unmerged == false) {
 					/* Print merged arena stats. */
 					malloc_cprintf(write_cb, cbopaque,
 					    "\nMerged arenas stats:\n");
 					stats_arena_print(write_cb, cbopaque,
-					    narenas);
+					    narenas, bins, large);
 				}
 			}
 		}
@@ -765,11 +525,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 
 			CTL_GET("arenas.narenas", &narenas, unsigned);
 			{
-				bool initialized[narenas];
+				VARIABLE_ARRAY(bool, initialized, narenas);
 				size_t isz;
 				unsigned i;
 
-				isz = sizeof(initialized);
+				isz = sizeof(bool) * narenas;
 				xmallctl("arenas.initialized", initialized,
 				    &isz, NULL, 0);
 
@@ -779,12 +539,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 						    cbopaque,
 						    "\narenas[%u]:\n", i);
 						stats_arena_print(write_cb,
-						    cbopaque, i);
+						    cbopaque, i, bins, large);
 					}
 				}
 			}
 		}
 	}
-#endif /* #ifdef JEMALLOC_STATS */
-	write_cb(cbopaque, "--- End jemalloc statistics ---\n");
+	malloc_cprintf(write_cb, cbopaque, "--- End jemalloc statistics ---\n");
 }

src/tcache.c

@@ -1,70 +1,92 @@
 #define	JEMALLOC_TCACHE_C_
 #include "jemalloc/internal/jemalloc_internal.h"
-#ifdef JEMALLOC_TCACHE
+
 /******************************************************************************/
 /* Data. */
 
+malloc_tsd_data(, tcache, tcache_t *, NULL)
+malloc_tsd_data(, tcache_enabled, tcache_enabled_t, tcache_enabled_default)
+
 bool	opt_tcache = true;
 ssize_t	opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
-ssize_t	opt_lg_tcache_gc_sweep = LG_TCACHE_GC_SWEEP_DEFAULT;
 
 tcache_bin_info_t	*tcache_bin_info;
 static unsigned		stack_nelms; /* Total stack elms per tcache. */
 
-/* Map of thread-specific caches. */
-#ifndef NO_TLS
-__thread tcache_t	*tcache_tls JEMALLOC_ATTR(tls_model("initial-exec"));
-#endif
-
-/*
- * Same contents as tcache, but initialized such that the TSD destructor is
- * called when a thread exits, so that the cache can be cleaned up.
- */
-pthread_key_t		tcache_tsd;
-
-size_t				nhbins;
-size_t				tcache_maxclass;
-unsigned			tcache_gc_incr;
+size_t			nhbins;
+size_t			tcache_maxclass;
 
 /******************************************************************************/
-/* Function prototypes for non-inline static functions. */
 
-static void	tcache_thread_cleanup(void *arg);
+size_t	tcache_salloc(const void *ptr)
+{
 
-/******************************************************************************/
+	return (arena_salloc(ptr, false));
+}
+
+void
+tcache_event_hard(tcache_t *tcache)
+{
+	size_t binind = tcache->next_gc_bin;
+	tcache_bin_t *tbin = &tcache->tbins[binind];
+	tcache_bin_info_t *tbin_info = &tcache_bin_info[binind];
+
+	if (tbin->low_water > 0) {
+		/*
+		 * Flush (ceiling) 3/4 of the objects below the low water mark.
+		 */
+		if (binind < NBINS) {
+			tcache_bin_flush_small(tbin, binind, tbin->ncached -
+			    tbin->low_water + (tbin->low_water >> 2), tcache);
+		} else {
+			tcache_bin_flush_large(tbin, binind, tbin->ncached -
+			    tbin->low_water + (tbin->low_water >> 2), tcache);
+		}
+		/*
+		 * Reduce fill count by 2X.  Limit lg_fill_div such that the
+		 * fill count is always at least 1.
+		 */
+		if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) >= 1)
+			tbin->lg_fill_div++;
+	} else if (tbin->low_water < 0) {
+		/*
+		 * Increase fill count by 2X.  Make sure lg_fill_div stays
+		 * greater than 0.
+		 */
+		if (tbin->lg_fill_div > 1)
+			tbin->lg_fill_div--;
+	}
+	tbin->low_water = tbin->ncached;
+
+	tcache->next_gc_bin++;
+	if (tcache->next_gc_bin == nhbins)
+		tcache->next_gc_bin = 0;
+	tcache->ev_cnt = 0;
+}
 
 void *
 tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
 {
 	void *ret;
 
-	arena_tcache_fill_small(tcache->arena, tbin, binind
-#ifdef JEMALLOC_PROF
-	    , tcache->prof_accumbytes
-#endif
-	    );
-#ifdef JEMALLOC_PROF
-	tcache->prof_accumbytes = 0;
-#endif
+	arena_tcache_fill_small(tcache->arena, tbin, binind,
+	    config_prof ? tcache->prof_accumbytes : 0);
+	if (config_prof)
+		tcache->prof_accumbytes = 0;
 	ret = tcache_alloc_easy(tbin);
 
 	return (ret);
 }
 
 void
-tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-    , tcache_t *tcache
-#endif
-    )
+tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem,
+    tcache_t *tcache)
 {
 	void *ptr;
 	unsigned i, nflush, ndeferred;
-#ifdef JEMALLOC_STATS
 	bool merged_stats = false;
-#endif
 
-	assert(binind < nbins);
+	assert(binind < NBINS);
 	assert(rem <= tbin->ncached);
 
 	for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
@@ -74,25 +96,20 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
 		arena_t *arena = chunk->arena;
 		arena_bin_t *bin = &arena->bins[binind];
 
-#ifdef JEMALLOC_PROF
-		if (arena == tcache->arena) {
-			malloc_mutex_lock(&arena->lock);
-			arena_prof_accum(arena, tcache->prof_accumbytes);
-			malloc_mutex_unlock(&arena->lock);
+		if (config_prof && arena == tcache->arena) {
+			if (arena_prof_accum(arena, tcache->prof_accumbytes))
+				prof_idump();
 			tcache->prof_accumbytes = 0;
 		}
-#endif
 
 		malloc_mutex_lock(&bin->lock);
-#ifdef JEMALLOC_STATS
-		if (arena == tcache->arena) {
+		if (config_stats && arena == tcache->arena) {
 			assert(merged_stats == false);
 			merged_stats = true;
 			bin->stats.nflushes++;
 			bin->stats.nrequests += tbin->tstats.nrequests;
 			tbin->tstats.nrequests = 0;
 		}
-#endif
 		ndeferred = 0;
 		for (i = 0; i < nflush; i++) {
 			ptr = tbin->avail[i];
@@ -100,10 +117,15 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
 			chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 			if (chunk->arena == arena) {
 				size_t pageind = ((uintptr_t)ptr -
-				    (uintptr_t)chunk) >> PAGE_SHIFT;
+				    (uintptr_t)chunk) >> LG_PAGE;
 				arena_chunk_map_t *mapelm =
-				    &chunk->map[pageind-map_bias];
-				arena_dalloc_bin(arena, chunk, ptr, mapelm);
+				    arena_mapp_get(chunk, pageind);
+				if (config_fill && opt_junk) {
+					arena_alloc_junk_small(ptr,
+					    &arena_bin_info[binind], true);
+				}
+				arena_dalloc_bin_locked(arena, chunk, ptr,
+				    mapelm);
 			} else {
 				/*
 				 * This object was allocated via a different
@@ -117,8 +139,7 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
 		}
 		malloc_mutex_unlock(&bin->lock);
 	}
-#ifdef JEMALLOC_STATS
-	if (merged_stats == false) {
+	if (config_stats && merged_stats == false) {
 		/*
 		 * The flush loop didn't happen to flush to this thread's
 		 * arena, so the stats didn't get merged.  Manually do so now.
@@ -130,7 +151,6 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
 		tbin->tstats.nrequests = 0;
 		malloc_mutex_unlock(&bin->lock);
 	}
-#endif
 
 	memmove(tbin->avail, &tbin->avail[tbin->ncached - rem],
 	    rem * sizeof(void *));
@@ -140,17 +160,12 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
 }
 
 void
-tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-    , tcache_t *tcache
-#endif
-    )
+tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
+    tcache_t *tcache)
 {
 	void *ptr;
 	unsigned i, nflush, ndeferred;
-#ifdef JEMALLOC_STATS
 	bool merged_stats = false;
-#endif
 
 	assert(binind < nhbins);
 	assert(rem <= tbin->ncached);
@@ -160,32 +175,33 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem
 		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
 		    tbin->avail[0]);
 		arena_t *arena = chunk->arena;
+		UNUSED bool idump;
 
+		if (config_prof)
+			idump = false;
 		malloc_mutex_lock(&arena->lock);
-#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS))
-		if (arena == tcache->arena) {
-#endif
-#ifdef JEMALLOC_PROF
-			arena_prof_accum(arena, tcache->prof_accumbytes);
-			tcache->prof_accumbytes = 0;
-#endif
-#ifdef JEMALLOC_STATS
-			merged_stats = true;
-			arena->stats.nrequests_large += tbin->tstats.nrequests;
-			arena->stats.lstats[binind - nbins].nrequests +=
-			    tbin->tstats.nrequests;
-			tbin->tstats.nrequests = 0;
-#endif
-#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS))
+		if ((config_prof || config_stats) && arena == tcache->arena) {
+			if (config_prof) {
+				idump = arena_prof_accum_locked(arena,
+				    tcache->prof_accumbytes);
+				tcache->prof_accumbytes = 0;
+			}
+			if (config_stats) {
+				merged_stats = true;
+				arena->stats.nrequests_large +=
+				    tbin->tstats.nrequests;
+				arena->stats.lstats[binind - NBINS].nrequests +=
+				    tbin->tstats.nrequests;
+				tbin->tstats.nrequests = 0;
+			}
 		}
-#endif
 		ndeferred = 0;
 		for (i = 0; i < nflush; i++) {
 			ptr = tbin->avail[i];
 			assert(ptr != NULL);
 			chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 			if (chunk->arena == arena)
-				arena_dalloc_large(arena, chunk, ptr);
+				arena_dalloc_large_locked(arena, chunk, ptr);
 			else {
 				/*
 				 * This object was allocated via a different
@@ -198,9 +214,10 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem
 			}
 		}
 		malloc_mutex_unlock(&arena->lock);
+		if (config_prof && idump)
+			prof_idump();
 	}
-#ifdef JEMALLOC_STATS
-	if (merged_stats == false) {
+	if (config_stats && merged_stats == false) {
 		/*
 		 * The flush loop didn't happen to flush to this thread's
 		 * arena, so the stats didn't get merged.  Manually do so now.
@@ -208,12 +225,11 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem
 		arena_t *arena = tcache->arena;
 		malloc_mutex_lock(&arena->lock);
 		arena->stats.nrequests_large += tbin->tstats.nrequests;
-		arena->stats.lstats[binind - nbins].nrequests +=
+		arena->stats.lstats[binind - NBINS].nrequests +=
 		    tbin->tstats.nrequests;
 		tbin->tstats.nrequests = 0;
 		malloc_mutex_unlock(&arena->lock);
 	}
-#endif
 
 	memmove(tbin->avail, &tbin->avail[tbin->ncached - rem],
 	    rem * sizeof(void *));
@@ -222,6 +238,33 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem
 		tbin->low_water = tbin->ncached;
 }
 
+void
+tcache_arena_associate(tcache_t *tcache, arena_t *arena)
+{
+
+	if (config_stats) {
+		/* Link into list of extant tcaches. */
+		malloc_mutex_lock(&arena->lock);
+		ql_elm_new(tcache, link);
+		ql_tail_insert(&arena->tcache_ql, tcache, link);
+		malloc_mutex_unlock(&arena->lock);
+	}
+	tcache->arena = arena;
+}
+
+void
+tcache_arena_dissociate(tcache_t *tcache)
+{
+
+	if (config_stats) {
+		/* Unlink from list of extant tcaches. */
+		malloc_mutex_lock(&tcache->arena->lock);
+		ql_remove(&tcache->arena->tcache_ql, tcache, link);
+		tcache_stats_merge(tcache, tcache->arena);
+		malloc_mutex_unlock(&tcache->arena->lock);
+	}
+}
+
 tcache_t *
 tcache_create(arena_t *arena)
 {
@@ -244,25 +287,18 @@ tcache_create(arena_t *arena)
 	 */
 	size = (size + CACHELINE_MASK) & (-CACHELINE);
 
-	if (size <= small_maxclass)
+	if (size <= SMALL_MAXCLASS)
 		tcache = (tcache_t *)arena_malloc_small(arena, size, true);
 	else if (size <= tcache_maxclass)
 		tcache = (tcache_t *)arena_malloc_large(arena, size, true);
 	else
-		tcache = (tcache_t *)icalloc(size);
+		tcache = (tcache_t *)icalloct(size, false, arena);
 
 	if (tcache == NULL)
 		return (NULL);
 
-#ifdef JEMALLOC_STATS
-	/* Link into list of extant tcaches. */
-	malloc_mutex_lock(&arena->lock);
-	ql_elm_new(tcache, link);
-	ql_tail_insert(&arena->tcache_ql, tcache, link);
-	malloc_mutex_unlock(&arena->lock);
-#endif
+	tcache_arena_associate(tcache, arena);
 
-	tcache->arena = arena;
 	assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
 	for (i = 0; i < nhbins; i++) {
 		tcache->tbins[i].lg_fill_div = 1;
@@ -271,7 +307,7 @@ tcache_create(arena_t *arena)
 		stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
 	}
 
-	TCACHE_SET(tcache);
+	tcache_tsd_set(&tcache);
 
 	return (tcache);
 }
@@ -282,121 +318,97 @@ tcache_destroy(tcache_t *tcache)
 	unsigned i;
 	size_t tcache_size;
 
-#ifdef JEMALLOC_STATS
-	/* Unlink from list of extant tcaches. */
-	malloc_mutex_lock(&tcache->arena->lock);
-	ql_remove(&tcache->arena->tcache_ql, tcache, link);
-	malloc_mutex_unlock(&tcache->arena->lock);
-	tcache_stats_merge(tcache, tcache->arena);
-#endif
+	tcache_arena_dissociate(tcache);
 
-	for (i = 0; i < nbins; i++) {
+	for (i = 0; i < NBINS; i++) {
 		tcache_bin_t *tbin = &tcache->tbins[i];
-		tcache_bin_flush_small(tbin, i, 0
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-		    , tcache
-#endif
-		    );
+		tcache_bin_flush_small(tbin, i, 0, tcache);
 
-#ifdef JEMALLOC_STATS
-		if (tbin->tstats.nrequests != 0) {
+		if (config_stats && tbin->tstats.nrequests != 0) {
 			arena_t *arena = tcache->arena;
 			arena_bin_t *bin = &arena->bins[i];
 			malloc_mutex_lock(&bin->lock);
 			bin->stats.nrequests += tbin->tstats.nrequests;
 			malloc_mutex_unlock(&bin->lock);
 		}
-#endif
 	}
 
 	for (; i < nhbins; i++) {
 		tcache_bin_t *tbin = &tcache->tbins[i];
-		tcache_bin_flush_large(tbin, i, 0
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-		    , tcache
-#endif
-		    );
+		tcache_bin_flush_large(tbin, i, 0, tcache);
 
-#ifdef JEMALLOC_STATS
-		if (tbin->tstats.nrequests != 0) {
+		if (config_stats && tbin->tstats.nrequests != 0) {
 			arena_t *arena = tcache->arena;
 			malloc_mutex_lock(&arena->lock);
 			arena->stats.nrequests_large += tbin->tstats.nrequests;
-			arena->stats.lstats[i - nbins].nrequests +=
+			arena->stats.lstats[i - NBINS].nrequests +=
 			    tbin->tstats.nrequests;
 			malloc_mutex_unlock(&arena->lock);
 		}
-#endif
 	}
 
-#ifdef JEMALLOC_PROF
-	if (tcache->prof_accumbytes > 0) {
-		malloc_mutex_lock(&tcache->arena->lock);
-		arena_prof_accum(tcache->arena, tcache->prof_accumbytes);
-		malloc_mutex_unlock(&tcache->arena->lock);
-	}
-#endif
+	if (config_prof && tcache->prof_accumbytes > 0 &&
+	    arena_prof_accum(tcache->arena, tcache->prof_accumbytes))
+		prof_idump();
 
-	tcache_size = arena_salloc(tcache);
-	if (tcache_size <= small_maxclass) {
+	tcache_size = arena_salloc(tcache, false);
+	if (tcache_size <= SMALL_MAXCLASS) {
 		arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
 		arena_t *arena = chunk->arena;
 		size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >>
-		    PAGE_SHIFT;
-		arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias];
-		arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
-		    (uintptr_t)((pageind - (mapelm->bits >> PAGE_SHIFT)) <<
-		    PAGE_SHIFT));
-		arena_bin_t *bin = run->bin;
+		    LG_PAGE;
+		arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind);
 
-		malloc_mutex_lock(&bin->lock);
-		arena_dalloc_bin(arena, chunk, tcache, mapelm);
-		malloc_mutex_unlock(&bin->lock);
+		arena_dalloc_bin(arena, chunk, tcache, pageind, mapelm);
 	} else if (tcache_size <= tcache_maxclass) {
 		arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
 		arena_t *arena = chunk->arena;
 
-		malloc_mutex_lock(&arena->lock);
 		arena_dalloc_large(arena, chunk, tcache);
-		malloc_mutex_unlock(&arena->lock);
 	} else
-		idalloc(tcache);
+		idalloct(tcache, false);
 }
 
-static void
+void
 tcache_thread_cleanup(void *arg)
 {
-	tcache_t *tcache = (tcache_t *)arg;
+	tcache_t *tcache = *(tcache_t **)arg;
 
-	if (tcache == (void *)(uintptr_t)1) {
-		/*
-		 * The previous time this destructor was called, we set the key
-		 * to 1 so that other destructors wouldn't cause re-creation of
-		 * the tcache.  This time, do nothing, so that the destructor
-		 * will not be called again.
-		 */
-	} else if (tcache == (void *)(uintptr_t)2) {
+	if (tcache == TCACHE_STATE_DISABLED) {
+		/* Do nothing. */
+	} else if (tcache == TCACHE_STATE_REINCARNATED) {
 		/*
 		 * Another destructor called an allocator function after this
-		 * destructor was called.  Reset tcache to 1 in order to
-		 * receive another callback.
+		 * destructor was called.  Reset tcache to
+		 * TCACHE_STATE_PURGATORY in order to receive another callback.
+		 */
+		tcache = TCACHE_STATE_PURGATORY;
+		tcache_tsd_set(&tcache);
+	} else if (tcache == TCACHE_STATE_PURGATORY) {
+		/*
+		 * The previous time this destructor was called, we set the key
+		 * to TCACHE_STATE_PURGATORY so that other destructors wouldn't
+		 * cause re-creation of the tcache.  This time, do nothing, so
+		 * that the destructor will not be called again.
 		 */
-		TCACHE_SET((uintptr_t)1);
 	} else if (tcache != NULL) {
-		assert(tcache != (void *)(uintptr_t)1);
+		assert(tcache != TCACHE_STATE_PURGATORY);
 		tcache_destroy(tcache);
-		TCACHE_SET((uintptr_t)1);
+		tcache = TCACHE_STATE_PURGATORY;
+		tcache_tsd_set(&tcache);
 	}
 }
 
-#ifdef JEMALLOC_STATS
+/* Caller must own arena->lock. */
 void
 tcache_stats_merge(tcache_t *tcache, arena_t *arena)
 {
 	unsigned i;
 
+	cassert(config_stats);
+
 	/* Merge and reset tcache stats. */
-	for (i = 0; i < nbins; i++) {
+	for (i = 0; i < NBINS; i++) {
 		arena_bin_t *bin = &arena->bins[i];
 		tcache_bin_t *tbin = &tcache->tbins[i];
 		malloc_mutex_lock(&bin->lock);
@@ -406,75 +418,62 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena)
 	}
 
 	for (; i < nhbins; i++) {
-		malloc_large_stats_t *lstats = &arena->stats.lstats[i - nbins];
+		malloc_large_stats_t *lstats = &arena->stats.lstats[i - NBINS];
 		tcache_bin_t *tbin = &tcache->tbins[i];
 		arena->stats.nrequests_large += tbin->tstats.nrequests;
 		lstats->nrequests += tbin->tstats.nrequests;
 		tbin->tstats.nrequests = 0;
 	}
 }
-#endif
 
 bool
-tcache_boot(void)
+tcache_boot0(void)
 {
+	unsigned i;
 
-	if (opt_tcache) {
-		unsigned i;
+	/*
+	 * If necessary, clamp opt_lg_tcache_max, now that arena_maxclass is
+	 * known.
+	 */
+	if (opt_lg_tcache_max < 0 || (1U << opt_lg_tcache_max) < SMALL_MAXCLASS)
+		tcache_maxclass = SMALL_MAXCLASS;
+	else if ((1U << opt_lg_tcache_max) > arena_maxclass)
+		tcache_maxclass = arena_maxclass;
+	else
+		tcache_maxclass = (1U << opt_lg_tcache_max);
 
-		/*
-		 * If necessary, clamp opt_lg_tcache_max, now that
-		 * small_maxclass and arena_maxclass are known.
-		 */
-		if (opt_lg_tcache_max < 0 || (1U <<
-		    opt_lg_tcache_max) < small_maxclass)
-			tcache_maxclass = small_maxclass;
-		else if ((1U << opt_lg_tcache_max) > arena_maxclass)
-			tcache_maxclass = arena_maxclass;
-		else
-			tcache_maxclass = (1U << opt_lg_tcache_max);
+	nhbins = NBINS + (tcache_maxclass >> LG_PAGE);
 
-		nhbins = nbins + (tcache_maxclass >> PAGE_SHIFT);
-
-		/* Initialize tcache_bin_info. */
-		tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins *
-		    sizeof(tcache_bin_info_t));
-		if (tcache_bin_info == NULL)
-			return (true);
-		stack_nelms = 0;
-		for (i = 0; i < nbins; i++) {
-			if ((arena_bin_info[i].nregs << 1) <=
-			    TCACHE_NSLOTS_SMALL_MAX) {
-				tcache_bin_info[i].ncached_max =
-				    (arena_bin_info[i].nregs << 1);
-			} else {
-				tcache_bin_info[i].ncached_max =
-				    TCACHE_NSLOTS_SMALL_MAX;
-			}
-			stack_nelms += tcache_bin_info[i].ncached_max;
-		}
-		for (; i < nhbins; i++) {
-			tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE;
-			stack_nelms += tcache_bin_info[i].ncached_max;
-		}
-
-		/* Compute incremental GC event threshold. */
-		if (opt_lg_tcache_gc_sweep >= 0) {
-			tcache_gc_incr = ((1U << opt_lg_tcache_gc_sweep) /
-			    nbins) + (((1U << opt_lg_tcache_gc_sweep) % nbins ==
-			    0) ? 0 : 1);
-		} else
-			tcache_gc_incr = 0;
-
-		if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) !=
-		    0) {
-			malloc_write(
-			    "<jemalloc>: Error in pthread_key_create()\n");
-			abort();
+	/* Initialize tcache_bin_info. */
+	tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins *
+	    sizeof(tcache_bin_info_t));
+	if (tcache_bin_info == NULL)
+		return (true);
+	stack_nelms = 0;
+	for (i = 0; i < NBINS; i++) {
+		if ((arena_bin_info[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MAX) {
+			tcache_bin_info[i].ncached_max =
+			    (arena_bin_info[i].nregs << 1);
+		} else {
+			tcache_bin_info[i].ncached_max =
+			    TCACHE_NSLOTS_SMALL_MAX;
 		}
+		stack_nelms += tcache_bin_info[i].ncached_max;
+	}
+	for (; i < nhbins; i++) {
+		tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE;
+		stack_nelms += tcache_bin_info[i].ncached_max;
 	}
 
 	return (false);
 }
-/******************************************************************************/
-#endif /* JEMALLOC_TCACHE */
+
+bool
+tcache_boot1(void)
+{
+
+	if (tcache_tsd_boot() || tcache_enabled_tsd_boot())
+		return (true);
+
+	return (false);
+}

src/tsd.c

@@ -0,0 +1,141 @@
+#define	JEMALLOC_TSD_C_
+#include "jemalloc/internal/jemalloc_internal.h"
+
+/******************************************************************************/
+/* Data. */
+
+static unsigned ncleanups;
+static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX];
+
+/******************************************************************************/
+
+void *
+malloc_tsd_malloc(size_t size)
+{
+
+	/* Avoid choose_arena() in order to dodge bootstrapping issues. */
+	return (arena_malloc(arenas[0], size, false, false));
+}
+
+void
+malloc_tsd_dalloc(void *wrapper)
+{
+
+	idalloct(wrapper, false);
+}
+
+void
+malloc_tsd_no_cleanup(void *arg)
+{
+
+	not_reached();
+}
+
+#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32)
+#ifndef _WIN32
+JEMALLOC_EXPORT
+#endif
+void
+_malloc_thread_cleanup(void)
+{
+	bool pending[MALLOC_TSD_CLEANUPS_MAX], again;
+	unsigned i;
+
+	for (i = 0; i < ncleanups; i++)
+		pending[i] = true;
+
+	do {
+		again = false;
+		for (i = 0; i < ncleanups; i++) {
+			if (pending[i]) {
+				pending[i] = cleanups[i]();
+				if (pending[i])
+					again = true;
+			}
+		}
+	} while (again);
+}
+#endif
+
+void
+malloc_tsd_cleanup_register(bool (*f)(void))
+{
+
+	assert(ncleanups < MALLOC_TSD_CLEANUPS_MAX);
+	cleanups[ncleanups] = f;
+	ncleanups++;
+}
+
+void
+malloc_tsd_boot(void)
+{
+
+	ncleanups = 0;
+}
+
+#ifdef _WIN32
+static BOOL WINAPI
+_tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved)
+{
+
+	switch (fdwReason) {
+#ifdef JEMALLOC_LAZY_LOCK
+	case DLL_THREAD_ATTACH:
+		isthreaded = true;
+		break;
+#endif
+	case DLL_THREAD_DETACH:
+		_malloc_thread_cleanup();
+		break;
+	default:
+		break;
+	}
+	return (true);
+}
+
+#ifdef _MSC_VER
+#  ifdef _M_IX86
+#    pragma comment(linker, "/INCLUDE:__tls_used")
+#  else
+#    pragma comment(linker, "/INCLUDE:_tls_used")
+#  endif
+#  pragma section(".CRT$XLY",long,read)
+#endif
+JEMALLOC_SECTION(".CRT$XLY") JEMALLOC_ATTR(used)
+static const BOOL	(WINAPI *tls_callback)(HINSTANCE hinstDLL,
+    DWORD fdwReason, LPVOID lpvReserved) = _tls_callback;
+#endif
+
+#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
+    !defined(_WIN32))
+void *
+tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block)
+{
+	pthread_t self = pthread_self();
+	tsd_init_block_t *iter;
+
+	/* Check whether this thread has already inserted into the list. */
+	malloc_mutex_lock(&head->lock);
+	ql_foreach(iter, &head->blocks, link) {
+		if (iter->thread == self) {
+			malloc_mutex_unlock(&head->lock);
+			return (iter->data);
+		}
+	}
+	/* Insert block into list. */
+	ql_elm_new(block, link);
+	block->thread = self;
+	ql_tail_insert(&head->blocks, block, link);
+	malloc_mutex_unlock(&head->lock);
+	return (NULL);
+}
+
+void
+tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block)
+{
+
+	malloc_mutex_lock(&head->lock);
+	ql_remove(&head->blocks, block, link);
+	malloc_mutex_unlock(&head->lock);
+}
+#endif

src/util.c

@@ -0,0 +1,648 @@
+#define	assert(e) do {							\
+	if (config_debug && !(e)) {					\
+		malloc_write("<jemalloc>: Failed assertion\n");		\
+		abort();						\
+	}								\
+} while (0)
+
+#define	not_reached() do {						\
+	if (config_debug) {						\
+		malloc_write("<jemalloc>: Unreachable code reached\n");	\
+		abort();						\
+	}								\
+} while (0)
+
+#define	not_implemented() do {						\
+	if (config_debug) {						\
+		malloc_write("<jemalloc>: Not implemented\n");		\
+		abort();						\
+	}								\
+} while (0)
+
+#define	JEMALLOC_UTIL_C_
+#include "jemalloc/internal/jemalloc_internal.h"
+
+/******************************************************************************/
+/* Function prototypes for non-inline static functions. */
+
+static void	wrtmessage(void *cbopaque, const char *s);
+#define	U2S_BUFSIZE	((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1)
+static char	*u2s(uintmax_t x, unsigned base, bool uppercase, char *s,
+    size_t *slen_p);
+#define	D2S_BUFSIZE	(1 + U2S_BUFSIZE)
+static char	*d2s(intmax_t x, char sign, char *s, size_t *slen_p);
+#define	O2S_BUFSIZE	(1 + U2S_BUFSIZE)
+static char	*o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p);
+#define	X2S_BUFSIZE	(2 + U2S_BUFSIZE)
+static char	*x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
+    size_t *slen_p);
+
+/******************************************************************************/
+
+/* malloc_message() setup. */
+static void
+wrtmessage(void *cbopaque, const char *s)
+{
+
+#ifdef SYS_write
+	/*
+	 * Use syscall(2) rather than write(2) when possible in order to avoid
+	 * the possibility of memory allocation within libc.  This is necessary
+	 * on FreeBSD; most operating systems do not have this problem though.
+	 */
+	UNUSED int result = syscall(SYS_write, STDERR_FILENO, s, strlen(s));
+#else
+	UNUSED int result = write(STDERR_FILENO, s, strlen(s));
+#endif
+}
+
+JEMALLOC_EXPORT void	(*je_malloc_message)(void *, const char *s);
+
+/*
+ * Wrapper around malloc_message() that avoids the need for
+ * je_malloc_message(...) throughout the code.
+ */
+void
+malloc_write(const char *s)
+{
+
+	if (je_malloc_message != NULL)
+		je_malloc_message(NULL, s);
+	else
+		wrtmessage(NULL, s);
+}
+
+/*
+ * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so
+ * provide a wrapper.
+ */
+int
+buferror(int err, char *buf, size_t buflen)
+{
+
+#ifdef _WIN32
+	FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(), 0,
+	    (LPSTR)buf, buflen, NULL);
+	return (0);
+#elif defined(_GNU_SOURCE)
+	char *b = strerror_r(err, buf, buflen);
+	if (b != buf) {
+		strncpy(buf, b, buflen);
+		buf[buflen-1] = '\0';
+	}
+	return (0);
+#else
+	return (strerror_r(err, buf, buflen));
+#endif
+}
+
+uintmax_t
+malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base)
+{
+	uintmax_t ret, digit;
+	int b;
+	bool neg;
+	const char *p, *ns;
+
+	p = nptr;
+	if (base < 0 || base == 1 || base > 36) {
+		ns = p;
+		set_errno(EINVAL);
+		ret = UINTMAX_MAX;
+		goto label_return;
+	}
+	b = base;
+
+	/* Swallow leading whitespace and get sign, if any. */
+	neg = false;
+	while (true) {
+		switch (*p) {
+		case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
+			p++;
+			break;
+		case '-':
+			neg = true;
+			/* Fall through. */
+		case '+':
+			p++;
+			/* Fall through. */
+		default:
+			goto label_prefix;
+		}
+	}
+
+	/* Get prefix, if any. */
+	label_prefix:
+	/*
+	 * Note where the first non-whitespace/sign character is so that it is
+	 * possible to tell whether any digits are consumed (e.g., "  0" vs.
+	 * "  -x").
+	 */
+	ns = p;
+	if (*p == '0') {
+		switch (p[1]) {
+		case '0': case '1': case '2': case '3': case '4': case '5':
+		case '6': case '7':
+			if (b == 0)
+				b = 8;
+			if (b == 8)
+				p++;
+			break;
+		case 'X': case 'x':
+			switch (p[2]) {
+			case '0': case '1': case '2': case '3': case '4':
+			case '5': case '6': case '7': case '8': case '9':
+			case 'A': case 'B': case 'C': case 'D': case 'E':
+			case 'F':
+			case 'a': case 'b': case 'c': case 'd': case 'e':
+			case 'f':
+				if (b == 0)
+					b = 16;
+				if (b == 16)
+					p += 2;
+				break;
+			default:
+				break;
+			}
+			break;
+		default:
+			p++;
+			ret = 0;
+			goto label_return;
+		}
+	}
+	if (b == 0)
+		b = 10;
+
+	/* Convert. */
+	ret = 0;
+	while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b)
+	    || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b)
+	    || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) {
+		uintmax_t pret = ret;
+		ret *= b;
+		ret += digit;
+		if (ret < pret) {
+			/* Overflow. */
+			set_errno(ERANGE);
+			ret = UINTMAX_MAX;
+			goto label_return;
+		}
+		p++;
+	}
+	if (neg)
+		ret = -ret;
+
+	if (p == ns) {
+		/* No conversion performed. */
+		set_errno(EINVAL);
+		ret = UINTMAX_MAX;
+		goto label_return;
+	}
+
+label_return:
+	if (endptr != NULL) {
+		if (p == ns) {
+			/* No characters were converted. */
+			*endptr = (char *)nptr;
+		} else
+			*endptr = (char *)p;
+	}
+	return (ret);
+}
+
+static char *
+u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p)
+{
+	unsigned i;
+
+	i = U2S_BUFSIZE - 1;
+	s[i] = '\0';
+	switch (base) {
+	case 10:
+		do {
+			i--;
+			s[i] = "0123456789"[x % (uint64_t)10];
+			x /= (uint64_t)10;
+		} while (x > 0);
+		break;
+	case 16: {
+		const char *digits = (uppercase)
+		    ? "0123456789ABCDEF"
+		    : "0123456789abcdef";
+
+		do {
+			i--;
+			s[i] = digits[x & 0xf];
+			x >>= 4;
+		} while (x > 0);
+		break;
+	} default: {
+		const char *digits = (uppercase)
+		    ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+		    : "0123456789abcdefghijklmnopqrstuvwxyz";
+
+		assert(base >= 2 && base <= 36);
+		do {
+			i--;
+			s[i] = digits[x % (uint64_t)base];
+			x /= (uint64_t)base;
+		} while (x > 0);
+	}}
+
+	*slen_p = U2S_BUFSIZE - 1 - i;
+	return (&s[i]);
+}
+
+static char *
+d2s(intmax_t x, char sign, char *s, size_t *slen_p)
+{
+	bool neg;
+
+	if ((neg = (x < 0)))
+		x = -x;
+	s = u2s(x, 10, false, s, slen_p);
+	if (neg)
+		sign = '-';
+	switch (sign) {
+	case '-':
+		if (neg == false)
+			break;
+		/* Fall through. */
+	case ' ':
+	case '+':
+		s--;
+		(*slen_p)++;
+		*s = sign;
+		break;
+	default: not_reached();
+	}
+	return (s);
+}
+
+static char *
+o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p)
+{
+
+	s = u2s(x, 8, false, s, slen_p);
+	if (alt_form && *s != '0') {
+		s--;
+		(*slen_p)++;
+		*s = '0';
+	}
+	return (s);
+}
+
+static char *
+x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p)
+{
+
+	s = u2s(x, 16, uppercase, s, slen_p);
+	if (alt_form) {
+		s -= 2;
+		(*slen_p) += 2;
+		memcpy(s, uppercase ? "0X" : "0x", 2);
+	}
+	return (s);
+}
+
+int
+malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
+{
+	int ret;
+	size_t i;
+	const char *f;
+
+#define	APPEND_C(c) do {						\
+	if (i < size)							\
+		str[i] = (c);						\
+	i++;								\
+} while (0)
+#define	APPEND_S(s, slen) do {						\
+	if (i < size) {							\
+		size_t cpylen = (slen <= size - i) ? slen : size - i;	\
+		memcpy(&str[i], s, cpylen);				\
+	}								\
+	i += slen;							\
+} while (0)
+#define	APPEND_PADDED_S(s, slen, width, left_justify) do {		\
+	/* Left padding. */						\
+	size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ?	\
+	    (size_t)width - slen : 0);					\
+	if (left_justify == false && pad_len != 0) {			\
+		size_t j;						\
+		for (j = 0; j < pad_len; j++)				\
+			APPEND_C(' ');					\
+	}								\
+	/* Value. */							\
+	APPEND_S(s, slen);						\
+	/* Right padding. */						\
+	if (left_justify && pad_len != 0) {				\
+		size_t j;						\
+		for (j = 0; j < pad_len; j++)				\
+			APPEND_C(' ');					\
+	}								\
+} while (0)
+#define	GET_ARG_NUMERIC(val, len) do {					\
+	switch (len) {							\
+	case '?':							\
+		val = va_arg(ap, int);					\
+		break;							\
+	case '?' | 0x80:						\
+		val = va_arg(ap, unsigned int);				\
+		break;							\
+	case 'l':							\
+		val = va_arg(ap, long);					\
+		break;							\
+	case 'l' | 0x80:						\
+		val = va_arg(ap, unsigned long);			\
+		break;							\
+	case 'q':							\
+		val = va_arg(ap, long long);				\
+		break;							\
+	case 'q' | 0x80:						\
+		val = va_arg(ap, unsigned long long);			\
+		break;							\
+	case 'j':							\
+		val = va_arg(ap, intmax_t);				\
+		break;							\
+	case 'j' | 0x80:						\
+		val = va_arg(ap, uintmax_t);				\
+		break;							\
+	case 't':							\
+		val = va_arg(ap, ptrdiff_t);				\
+		break;							\
+	case 'z':							\
+		val = va_arg(ap, ssize_t);				\
+		break;							\
+	case 'z' | 0x80:						\
+		val = va_arg(ap, size_t);				\
+		break;							\
+	case 'p': /* Synthetic; used for %p. */				\
+		val = va_arg(ap, uintptr_t);				\
+		break;							\
+	default: not_reached();						\
+	}								\
+} while (0)
+
+	i = 0;
+	f = format;
+	while (true) {
+		switch (*f) {
+		case '\0': goto label_out;
+		case '%': {
+			bool alt_form = false;
+			bool left_justify = false;
+			bool plus_space = false;
+			bool plus_plus = false;
+			int prec = -1;
+			int width = -1;
+			unsigned char len = '?';
+
+			f++;
+			/* Flags. */
+			while (true) {
+				switch (*f) {
+				case '#':
+					assert(alt_form == false);
+					alt_form = true;
+					break;
+				case '-':
+					assert(left_justify == false);
+					left_justify = true;
+					break;
+				case ' ':
+					assert(plus_space == false);
+					plus_space = true;
+					break;
+				case '+':
+					assert(plus_plus == false);
+					plus_plus = true;
+					break;
+				default: goto label_width;
+				}
+				f++;
+			}
+			/* Width. */
+			label_width:
+			switch (*f) {
+			case '*':
+				width = va_arg(ap, int);
+				f++;
+				if (width < 0) {
+					left_justify = true;
+					width = -width;
+				}
+				break;
+			case '0': case '1': case '2': case '3': case '4':
+			case '5': case '6': case '7': case '8': case '9': {
+				uintmax_t uwidth;
+				set_errno(0);
+				uwidth = malloc_strtoumax(f, (char **)&f, 10);
+				assert(uwidth != UINTMAX_MAX || get_errno() !=
+				    ERANGE);
+				width = (int)uwidth;
+				break;
+			} default:
+				break;
+			}
+			/* Width/precision separator. */
+			if (*f == '.')
+				f++;
+			else
+				goto label_length;
+			/* Precision. */
+			switch (*f) {
+			case '*':
+				prec = va_arg(ap, int);
+				f++;
+				break;
+			case '0': case '1': case '2': case '3': case '4':
+			case '5': case '6': case '7': case '8': case '9': {
+				uintmax_t uprec;
+				set_errno(0);
+				uprec = malloc_strtoumax(f, (char **)&f, 10);
+				assert(uprec != UINTMAX_MAX || get_errno() !=
+				    ERANGE);
+				prec = (int)uprec;
+				break;
+			}
+			default: break;
+			}
+			/* Length. */
+			label_length:
+			switch (*f) {
+			case 'l':
+				f++;
+				if (*f == 'l') {
+					len = 'q';
+					f++;
+				} else
+					len = 'l';
+				break;
+			case 'q': case 'j': case 't': case 'z':
+				len = *f;
+				f++;
+				break;
+			default: break;
+			}
+			/* Conversion specifier. */
+			switch (*f) {
+				char *s;
+				size_t slen;
+			case '%':
+				/* %% */
+				APPEND_C(*f);
+				f++;
+				break;
+			case 'd': case 'i': {
+				intmax_t val JEMALLOC_CC_SILENCE_INIT(0);
+				char buf[D2S_BUFSIZE];
+
+				GET_ARG_NUMERIC(val, len);
+				s = d2s(val, (plus_plus ? '+' : (plus_space ?
+				    ' ' : '-')), buf, &slen);
+				APPEND_PADDED_S(s, slen, width, left_justify);
+				f++;
+				break;
+			} case 'o': {
+				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
+				char buf[O2S_BUFSIZE];
+
+				GET_ARG_NUMERIC(val, len | 0x80);
+				s = o2s(val, alt_form, buf, &slen);
+				APPEND_PADDED_S(s, slen, width, left_justify);
+				f++;
+				break;
+			} case 'u': {
+				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
+				char buf[U2S_BUFSIZE];
+
+				GET_ARG_NUMERIC(val, len | 0x80);
+				s = u2s(val, 10, false, buf, &slen);
+				APPEND_PADDED_S(s, slen, width, left_justify);
+				f++;
+				break;
+			} case 'x': case 'X': {
+				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
+				char buf[X2S_BUFSIZE];
+
+				GET_ARG_NUMERIC(val, len | 0x80);
+				s = x2s(val, alt_form, *f == 'X', buf, &slen);
+				APPEND_PADDED_S(s, slen, width, left_justify);
+				f++;
+				break;
+			} case 'c': {
+				unsigned char val;
+				char buf[2];
+
+				assert(len == '?' || len == 'l');
+				assert_not_implemented(len != 'l');
+				val = va_arg(ap, int);
+				buf[0] = val;
+				buf[1] = '\0';
+				APPEND_PADDED_S(buf, 1, width, left_justify);
+				f++;
+				break;
+			} case 's':
+				assert(len == '?' || len == 'l');
+				assert_not_implemented(len != 'l');
+				s = va_arg(ap, char *);
+				slen = (prec < 0) ? strlen(s) : prec;
+				APPEND_PADDED_S(s, slen, width, left_justify);
+				f++;
+				break;
+			case 'p': {
+				uintmax_t val;
+				char buf[X2S_BUFSIZE];
+
+				GET_ARG_NUMERIC(val, 'p');
+				s = x2s(val, true, false, buf, &slen);
+				APPEND_PADDED_S(s, slen, width, left_justify);
+				f++;
+				break;
+			} default: not_reached();
+			}
+			break;
+		} default: {
+			APPEND_C(*f);
+			f++;
+			break;
+		}}
+	}
+	label_out:
+	if (i < size)
+		str[i] = '\0';
+	else
+		str[size - 1] = '\0';
+	ret = i;
+
+#undef APPEND_C
+#undef APPEND_S
+#undef APPEND_PADDED_S
+#undef GET_ARG_NUMERIC
+	return (ret);
+}
+
+JEMALLOC_ATTR(format(printf, 3, 4))
+int
+malloc_snprintf(char *str, size_t size, const char *format, ...)
+{
+	int ret;
+	va_list ap;
+
+	va_start(ap, format);
+	ret = malloc_vsnprintf(str, size, format, ap);
+	va_end(ap);
+
+	return (ret);
+}
+
+void
+malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
+    const char *format, va_list ap)
+{
+	char buf[MALLOC_PRINTF_BUFSIZE];
+
+	if (write_cb == NULL) {
+		/*
+		 * The caller did not provide an alternate write_cb callback
+		 * function, so use the default one.  malloc_write() is an
+		 * inline function, so use malloc_message() directly here.
+		 */
+		write_cb = (je_malloc_message != NULL) ? je_malloc_message :
+		    wrtmessage;
+		cbopaque = NULL;
+	}
+
+	malloc_vsnprintf(buf, sizeof(buf), format, ap);
+	write_cb(cbopaque, buf);
+}
+
+/*
+ * Print to a callback function in such a way as to (hopefully) avoid memory
+ * allocation.
+ */
+JEMALLOC_ATTR(format(printf, 3, 4))
+void
+malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque,
+    const char *format, ...)
+{
+	va_list ap;
+
+	va_start(ap, format);
+	malloc_vcprintf(write_cb, cbopaque, format, ap);
+	va_end(ap);
+}
+
+/* Print to stderr in such a way as to avoid memory allocation. */
+JEMALLOC_ATTR(format(printf, 1, 2))
+void
+malloc_printf(const char *format, ...)
+{
+	va_list ap;
+
+	va_start(ap, format);
+	malloc_vcprintf(NULL, NULL, format, ap);
+	va_end(ap);
+}

src/zone.c

@@ -3,11 +3,18 @@
 #  error "This source file is for zones on Darwin (OS X)."
 #endif
 
+/*
+ * The malloc_default_purgeable_zone function is only available on >= 10.6.
+ * We need to check whether it is present at runtime, thus the weak_import.
+ */
+extern malloc_zone_t *malloc_default_purgeable_zone(void)
+JEMALLOC_ATTR(weak_import);
+
 /******************************************************************************/
 /* Data. */
 
-static malloc_zone_t zone, szone;
-static struct malloc_introspection_t zone_introspect, ozone_introspect;
+static malloc_zone_t zone;
+static struct malloc_introspection_t zone_introspect;
 
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
@@ -18,8 +25,10 @@ static void	*zone_calloc(malloc_zone_t *zone, size_t num, size_t size);
 static void	*zone_valloc(malloc_zone_t *zone, size_t size);
 static void	zone_free(malloc_zone_t *zone, void *ptr);
 static void	*zone_realloc(malloc_zone_t *zone, void *ptr, size_t size);
-#if (JEMALLOC_ZONE_VERSION >= 6)
+#if (JEMALLOC_ZONE_VERSION >= 5)
 static void	*zone_memalign(malloc_zone_t *zone, size_t alignment,
+#endif
+#if (JEMALLOC_ZONE_VERSION >= 6)
     size_t size);
 static void	zone_free_definite_size(malloc_zone_t *zone, void *ptr,
     size_t size);
@@ -28,19 +37,6 @@ static void	*zone_destroy(malloc_zone_t *zone);
 static size_t	zone_good_size(malloc_zone_t *zone, size_t size);
 static void	zone_force_lock(malloc_zone_t *zone);
 static void	zone_force_unlock(malloc_zone_t *zone);
-static size_t	ozone_size(malloc_zone_t *zone, void *ptr);
-static void	ozone_free(malloc_zone_t *zone, void *ptr);
-static void	*ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size);
-static unsigned	ozone_batch_malloc(malloc_zone_t *zone, size_t size,
-    void **results, unsigned num_requested);
-static void	ozone_batch_free(malloc_zone_t *zone, void **to_be_freed,
-    unsigned num);
-#if (JEMALLOC_ZONE_VERSION >= 6)
-static void	ozone_free_definite_size(malloc_zone_t *zone, void *ptr,
-    size_t size);
-#endif
-static void	ozone_force_lock(malloc_zone_t *zone);
-static void	ozone_force_unlock(malloc_zone_t *zone);
 
 /******************************************************************************/
 /*
@@ -60,21 +56,21 @@ zone_size(malloc_zone_t *zone, void *ptr)
 	 * not work in practice, we must check all pointers to assure that they
 	 * reside within a mapped chunk before determining size.
 	 */
-	return (ivsalloc(ptr));
+	return (ivsalloc(ptr, config_prof));
 }
 
 static void *
 zone_malloc(malloc_zone_t *zone, size_t size)
 {
 
-	return (JEMALLOC_P(malloc)(size));
+	return (je_malloc(size));
 }
 
 static void *
 zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
 {
 
-	return (JEMALLOC_P(calloc)(num, size));
+	return (je_calloc(num, size));
 }
 
 static void *
@@ -82,7 +78,7 @@ zone_valloc(malloc_zone_t *zone, size_t size)
 {
 	void *ret = NULL; /* Assignment avoids useless compiler warning. */
 
-	JEMALLOC_P(posix_memalign)(&ret, PAGE_SIZE, size);
+	je_posix_memalign(&ret, PAGE, size);
 
 	return (ret);
 }
@@ -91,33 +87,48 @@ static void
 zone_free(malloc_zone_t *zone, void *ptr)
 {
 
-	JEMALLOC_P(free)(ptr);
+	if (ivsalloc(ptr, config_prof) != 0) {
+		je_free(ptr);
+		return;
+	}
+
+	free(ptr);
 }
 
 static void *
 zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
 {
 
-	return (JEMALLOC_P(realloc)(ptr, size));
+	if (ivsalloc(ptr, config_prof) != 0)
+		return (je_realloc(ptr, size));
+
+	return (realloc(ptr, size));
 }
 
-#if (JEMALLOC_ZONE_VERSION >= 6)
+#if (JEMALLOC_ZONE_VERSION >= 5)
 static void *
 zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
 {
 	void *ret = NULL; /* Assignment avoids useless compiler warning. */
 
-	JEMALLOC_P(posix_memalign)(&ret, alignment, size);
+	je_posix_memalign(&ret, alignment, size);
 
 	return (ret);
 }
+#endif
 
+#if (JEMALLOC_ZONE_VERSION >= 6)
 static void
 zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
 {
 
-	assert(ivsalloc(ptr) == size);
-	JEMALLOC_P(free)(ptr);
+	if (ivsalloc(ptr, config_prof) != 0) {
+		assert(ivsalloc(ptr, config_prof) == size);
+		je_free(ptr);
+		return;
+	}
+
+	free(ptr);
 }
 #endif
 
@@ -126,29 +137,17 @@ zone_destroy(malloc_zone_t *zone)
 {
 
 	/* This function should never be called. */
-	assert(false);
+	not_reached();
 	return (NULL);
 }
 
 static size_t
 zone_good_size(malloc_zone_t *zone, size_t size)
 {
-	size_t ret;
-	void *p;
 
-	/*
-	 * Actually create an object of the appropriate size, then find out
-	 * how large it could have been without moving up to the next size
-	 * class.
-	 */
-	p = JEMALLOC_P(malloc)(size);
-	if (p != NULL) {
-		ret = isalloc(p);
-		JEMALLOC_P(free)(p);
-	} else
-		ret = size;
-
-	return (ret);
+	if (size == 0)
+		size = 1;
+	return (s2u(size));
 }
 
 static void
@@ -164,13 +163,24 @@ zone_force_unlock(malloc_zone_t *zone)
 {
 
 	if (isthreaded)
-		jemalloc_postfork();
+		jemalloc_postfork_parent();
 }
 
-malloc_zone_t *
-create_zone(void)
+JEMALLOC_ATTR(constructor)
+void
+register_zone(void)
 {
 
+	/*
+	 * If something else replaced the system default zone allocator, don't
+	 * register jemalloc's.
+	 */
+	malloc_zone_t *default_zone = malloc_default_zone();
+	if (!default_zone->zone_name ||
+	    strcmp(default_zone->zone_name, "DefaultMallocZone") != 0) {
+		return;
+	}
+
 	zone.size = (void *)zone_size;
 	zone.malloc = (void *)zone_malloc;
 	zone.calloc = (void *)zone_calloc;
@@ -183,10 +193,15 @@ create_zone(void)
 	zone.batch_free = NULL;
 	zone.introspect = &zone_introspect;
 	zone.version = JEMALLOC_ZONE_VERSION;
-#if (JEMALLOC_ZONE_VERSION >= 6)
+#if (JEMALLOC_ZONE_VERSION >= 5)
 	zone.memalign = zone_memalign;
+#endif
+#if (JEMALLOC_ZONE_VERSION >= 6)
 	zone.free_definite_size = zone_free_definite_size;
 #endif
+#if (JEMALLOC_ZONE_VERSION >= 8)
+	zone.pressure_relief = NULL;
+#endif
 
 	zone_introspect.enumerator = NULL;
 	zone_introspect.good_size = (void *)zone_good_size;
@@ -199,156 +214,45 @@ create_zone(void)
 #if (JEMALLOC_ZONE_VERSION >= 6)
 	zone_introspect.zone_locked = NULL;
 #endif
-
-	return (&zone);
-}
-
-static size_t
-ozone_size(malloc_zone_t *zone, void *ptr)
-{
-	size_t ret;
-
-	ret = ivsalloc(ptr);
-	if (ret == 0)
-		ret = szone.size(zone, ptr);
-
-	return (ret);
-}
-
-static void
-ozone_free(malloc_zone_t *zone, void *ptr)
-{
-
-	if (ivsalloc(ptr) != 0)
-		JEMALLOC_P(free)(ptr);
-	else {
-		size_t size = szone.size(zone, ptr);
-		if (size != 0)
-			(szone.free)(zone, ptr);
-	}
-}
-
-static void *
-ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
-{
-	size_t oldsize;
-
-	if (ptr == NULL)
-		return (JEMALLOC_P(malloc)(size));
-
-	oldsize = ivsalloc(ptr);
-	if (oldsize != 0)
-		return (JEMALLOC_P(realloc)(ptr, size));
-	else {
-		oldsize = szone.size(zone, ptr);
-		if (oldsize == 0)
-			return (JEMALLOC_P(malloc)(size));
-		else {
-			void *ret = JEMALLOC_P(malloc)(size);
-			if (ret != NULL) {
-				memcpy(ret, ptr, (oldsize < size) ? oldsize :
-				    size);
-				(szone.free)(zone, ptr);
-			}
-			return (ret);
-		}
-	}
-}
-
-static unsigned
-ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results,
-    unsigned num_requested)
-{
-
-	/* Don't bother implementing this interface, since it isn't required. */
-	return (0);
-}
-
-static void
-ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num)
-{
-	unsigned i;
-
-	for (i = 0; i < num; i++)
-		ozone_free(zone, to_be_freed[i]);
-}
-
-#if (JEMALLOC_ZONE_VERSION >= 6)
-static void
-ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
-{
-
-	if (ivsalloc(ptr) != 0) {
-		assert(ivsalloc(ptr) == size);
-		JEMALLOC_P(free)(ptr);
-	} else {
-		assert(size == szone.size(zone, ptr));
-		szone.free_definite_size(zone, ptr, size);
-	}
-}
+#if (JEMALLOC_ZONE_VERSION >= 7)
+	zone_introspect.enable_discharge_checking = NULL;
+	zone_introspect.disable_discharge_checking = NULL;
+	zone_introspect.discharge = NULL;
+#ifdef __BLOCKS__
+	zone_introspect.enumerate_discharged_pointers = NULL;
+#else
+	zone_introspect.enumerate_unavailable_without_blocks = NULL;
+#endif
 #endif
-
-static void
-ozone_force_lock(malloc_zone_t *zone)
-{
-
-	/* jemalloc locking is taken care of by the normal jemalloc zone. */
-	szone.introspect->force_lock(zone);
-}
-
-static void
-ozone_force_unlock(malloc_zone_t *zone)
-{
-
-	/* jemalloc locking is taken care of by the normal jemalloc zone. */
-	szone.introspect->force_unlock(zone);
-}
-
-/*
- * Overlay the default scalable zone (szone) such that existing allocations are
- * drained, and further allocations come from jemalloc.  This is necessary
- * because Core Foundation directly accesses and uses the szone before the
- * jemalloc library is even loaded.
- */
-void
-szone2ozone(malloc_zone_t *zone)
-{
 
 	/*
-	 * Stash a copy of the original szone so that we can call its
-	 * functions as needed.  Note that the internally, the szone stores its
-	 * bookkeeping data structures immediately following the malloc_zone_t
-	 * header, so when calling szone functions, we need to pass a pointer
-	 * to the original zone structure.
+	 * The default purgeable zone is created lazily by OSX's libc.  It uses
+	 * the default zone when it is created for "small" allocations
+	 * (< 15 KiB), but assumes the default zone is a scalable_zone.  This
+	 * obviously fails when the default zone is the jemalloc zone, so
+	 * malloc_default_purgeable_zone is called beforehand so that the
+	 * default purgeable zone is created when the default zone is still
+	 * a scalable_zone.  As purgeable zones only exist on >= 10.6, we need
+	 * to check for the existence of malloc_default_purgeable_zone() at
+	 * run time.
 	 */
-	memcpy(&szone, zone, sizeof(malloc_zone_t));
+	if (malloc_default_purgeable_zone != NULL)
+		malloc_default_purgeable_zone();
 
-	zone->size = (void *)ozone_size;
-	zone->malloc = (void *)zone_malloc;
-	zone->calloc = (void *)zone_calloc;
-	zone->valloc = (void *)zone_valloc;
-	zone->free = (void *)ozone_free;
-	zone->realloc = (void *)ozone_realloc;
-	zone->destroy = (void *)zone_destroy;
-	zone->zone_name = "jemalloc_ozone";
-	zone->batch_malloc = ozone_batch_malloc;
-	zone->batch_free = ozone_batch_free;
-	zone->introspect = &ozone_introspect;
-	zone->version = JEMALLOC_ZONE_VERSION;
-#if (JEMALLOC_ZONE_VERSION >= 6)
-	zone->memalign = zone_memalign;
-	zone->free_definite_size = ozone_free_definite_size;
-#endif
+	/* Register the custom zone.  At this point it won't be the default. */
+	malloc_zone_register(&zone);
 
-	ozone_introspect.enumerator = NULL;
-	ozone_introspect.good_size = (void *)zone_good_size;
-	ozone_introspect.check = NULL;
-	ozone_introspect.print = NULL;
-	ozone_introspect.log = NULL;
-	ozone_introspect.force_lock = (void *)ozone_force_lock;
-	ozone_introspect.force_unlock = (void *)ozone_force_unlock;
-	ozone_introspect.statistics = NULL;
-#if (JEMALLOC_ZONE_VERSION >= 6)
-	ozone_introspect.zone_locked = NULL;
-#endif
+	/*
+	 * Unregister and reregister the default zone.  On OSX >= 10.6,
+	 * unregistering takes the last registered zone and places it at the
+	 * location of the specified zone.  Unregistering the default zone thus
+	 * makes the last registered one the default.  On OSX < 10.6,
+	 * unregistering shifts all registered zones.  The first registered zone
+	 * then becomes the default.
+	 */
+	do {
+		default_zone = malloc_default_zone();
+		malloc_zone_unregister(default_zone);
+		malloc_zone_register(default_zone);
+	} while (malloc_default_zone() != &zone);
 }

test/allocated.c

@@ -1,142 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <stdbool.h>
-#include <pthread.h>
-#include <assert.h>
-#include <errno.h>
-#include <string.h>
-
-#define	JEMALLOC_MANGLE
-#include "jemalloc_test.h"
-
-void *
-thread_start(void *arg)
-{
-	int err;
-	void *p;
-	uint64_t a0, a1, d0, d1;
-	uint64_t *ap0, *ap1, *dp0, *dp1;
-	size_t sz, usize;
-
-	sz = sizeof(a0);
-	if ((err = JEMALLOC_P(mallctl)("thread.allocated", &a0, &sz, NULL,
-	    0))) {
-		if (err == ENOENT) {
-#ifdef JEMALLOC_STATS
-			assert(false);
-#endif
-			goto RETURN;
-		}
-		fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__,
-		    strerror(err));
-		exit(1);
-	}
-	sz = sizeof(ap0);
-	if ((err = JEMALLOC_P(mallctl)("thread.allocatedp", &ap0, &sz, NULL,
-	    0))) {
-		if (err == ENOENT) {
-#ifdef JEMALLOC_STATS
-			assert(false);
-#endif
-			goto RETURN;
-		}
-		fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__,
-		    strerror(err));
-		exit(1);
-	}
-	assert(*ap0 == a0);
-
-	sz = sizeof(d0);
-	if ((err = JEMALLOC_P(mallctl)("thread.deallocated", &d0, &sz, NULL,
-	    0))) {
-		if (err == ENOENT) {
-#ifdef JEMALLOC_STATS
-			assert(false);
-#endif
-			goto RETURN;
-		}
-		fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__,
-		    strerror(err));
-		exit(1);
-	}
-	sz = sizeof(dp0);
-	if ((err = JEMALLOC_P(mallctl)("thread.deallocatedp", &dp0, &sz, NULL,
-	    0))) {
-		if (err == ENOENT) {
-#ifdef JEMALLOC_STATS
-			assert(false);
-#endif
-			goto RETURN;
-		}
-		fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__,
-		    strerror(err));
-		exit(1);
-	}
-	assert(*dp0 == d0);
-
-	p = JEMALLOC_P(malloc)(1);
-	if (p == NULL) {
-		fprintf(stderr, "%s(): Error in malloc()\n", __func__);
-		exit(1);
-	}
-
-	sz = sizeof(a1);
-	JEMALLOC_P(mallctl)("thread.allocated", &a1, &sz, NULL, 0);
-	sz = sizeof(ap1);
-	JEMALLOC_P(mallctl)("thread.allocatedp", &ap1, &sz, NULL, 0);
-	assert(*ap1 == a1);
-	assert(ap0 == ap1);
-
-	usize = JEMALLOC_P(malloc_usable_size)(p);
-	assert(a0 + usize <= a1);
-
-	JEMALLOC_P(free)(p);
-
-	sz = sizeof(d1);
-	JEMALLOC_P(mallctl)("thread.deallocated", &d1, &sz, NULL, 0);
-	sz = sizeof(dp1);
-	JEMALLOC_P(mallctl)("thread.deallocatedp", &dp1, &sz, NULL, 0);
-	assert(*dp1 == d1);
-	assert(dp0 == dp1);
-
-	assert(d0 + usize <= d1);
-
-RETURN:
-	return (NULL);
-}
-
-int
-main(void)
-{
-	int ret = 0;
-	pthread_t thread;
-
-	fprintf(stderr, "Test begin\n");
-
-	thread_start(NULL);
-
-	if (pthread_create(&thread, NULL, thread_start, NULL)
-	    != 0) {
-		fprintf(stderr, "%s(): Error in pthread_create()\n", __func__);
-		ret = 1;
-		goto RETURN;
-	}
-	pthread_join(thread, (void *)&ret);
-
-	thread_start(NULL);
-
-	if (pthread_create(&thread, NULL, thread_start, NULL)
-	    != 0) {
-		fprintf(stderr, "%s(): Error in pthread_create()\n", __func__);
-		ret = 1;
-		goto RETURN;
-	}
-	pthread_join(thread, (void *)&ret);
-
-	thread_start(NULL);
-
-RETURN:
-	fprintf(stderr, "Test end\n");
-	return (ret);
-}

test/allocated.exp

@@ -1,2 +0,0 @@
-Test begin
-Test end

test/allocm.c

@@ -1,133 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-
-#define	JEMALLOC_MANGLE
-#include "jemalloc_test.h"
-
-#define CHUNK 0x400000
-/* #define MAXALIGN ((size_t)0x80000000000LLU) */
-#define MAXALIGN ((size_t)0x2000000LLU)
-#define NITER 4
-
-int
-main(void)
-{
-	int r;
-	void *p;
-	size_t sz, alignment, total, tsz;
-	unsigned i;
-	void *ps[NITER];
-
-	fprintf(stderr, "Test begin\n");
-
-	sz = 0;
-	r = JEMALLOC_P(allocm)(&p, &sz, 42, 0);
-	if (r != ALLOCM_SUCCESS) {
-		fprintf(stderr, "Unexpected allocm() error\n");
-		abort();
-	}
-	if (sz < 42)
-		fprintf(stderr, "Real size smaller than expected\n");
-	if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS)
-		fprintf(stderr, "Unexpected dallocm() error\n");
-
-	r = JEMALLOC_P(allocm)(&p, NULL, 42, 0);
-	if (r != ALLOCM_SUCCESS) {
-		fprintf(stderr, "Unexpected allocm() error\n");
-		abort();
-	}
-	if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS)
-		fprintf(stderr, "Unexpected dallocm() error\n");
-
-	r = JEMALLOC_P(allocm)(&p, NULL, 42, ALLOCM_ZERO);
-	if (r != ALLOCM_SUCCESS) {
-		fprintf(stderr, "Unexpected allocm() error\n");
-		abort();
-	}
-	if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS)
-		fprintf(stderr, "Unexpected dallocm() error\n");
-
-#if LG_SIZEOF_PTR == 3
-	alignment = 0x8000000000000000LLU;
-	sz        = 0x8000000000000000LLU;
-#else
-	alignment = 0x80000000LU;
-	sz        = 0x80000000LU;
-#endif
-	r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment));
-	if (r == ALLOCM_SUCCESS) {
-		fprintf(stderr,
-		    "Expected error for allocm(&p, %zu, 0x%x)\n",
-		    sz, ALLOCM_ALIGN(alignment));
-	}
-
-#if LG_SIZEOF_PTR == 3
-	alignment = 0x4000000000000000LLU;
-	sz        = 0x8400000000000001LLU;
-#else
-	alignment = 0x40000000LU;
-	sz        = 0x84000001LU;
-#endif
-	r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment));
-	if (r == ALLOCM_SUCCESS) {
-		fprintf(stderr,
-		    "Expected error for allocm(&p, %zu, 0x%x)\n",
-		    sz, ALLOCM_ALIGN(alignment));
-	}
-
-	alignment = 0x10LLU;
-#if LG_SIZEOF_PTR == 3
-	sz   = 0xfffffffffffffff0LLU;
-#else
-	sz   = 0xfffffff0LU;
-#endif
-	r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment));
-	if (r == ALLOCM_SUCCESS) {
-		fprintf(stderr,
-		    "Expected error for allocm(&p, %zu, 0x%x)\n",
-		    sz, ALLOCM_ALIGN(alignment));
-	}
-
-	for (i = 0; i < NITER; i++)
-		ps[i] = NULL;
-
-	for (alignment = 8;
-	    alignment <= MAXALIGN;
-	    alignment <<= 1) {
-		total = 0;
-		fprintf(stderr, "Alignment: %zu\n", alignment);
-		for (sz = 1;
-		    sz < 3 * alignment && sz < (1U << 31);
-		    sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) {
-			for (i = 0; i < NITER; i++) {
-				r = JEMALLOC_P(allocm)(&ps[i], NULL, sz,
-				    ALLOCM_ALIGN(alignment) | ALLOCM_ZERO);
-				if (r != ALLOCM_SUCCESS) {
-					fprintf(stderr,
-					    "Error for size %zu (0x%zx): %d\n",
-					    sz, sz, r);
-					exit(1);
-				}
-				if ((uintptr_t)p & (alignment-1)) {
-					fprintf(stderr,
-					    "%p inadequately aligned for"
-					    " alignment: %zu\n", p, alignment);
-				}
-				JEMALLOC_P(sallocm)(ps[i], &tsz, 0);
-				total += tsz;
-				if (total >= (MAXALIGN << 1))
-					break;
-			}
-			for (i = 0; i < NITER; i++) {
-				if (ps[i] != NULL) {
-					JEMALLOC_P(dallocm)(ps[i], 0);
-					ps[i] = NULL;
-				}
-			}
-		}
-	}
-
-	fprintf(stderr, "Test end\n");
-	return (0);
-}

test/allocm.exp

@@ -1,25 +0,0 @@
-Test begin
-Alignment: 8
-Alignment: 16
-Alignment: 32
-Alignment: 64
-Alignment: 128
-Alignment: 256
-Alignment: 512
-Alignment: 1024
-Alignment: 2048
-Alignment: 4096
-Alignment: 8192
-Alignment: 16384
-Alignment: 32768
-Alignment: 65536
-Alignment: 131072
-Alignment: 262144
-Alignment: 524288
-Alignment: 1048576
-Alignment: 2097152
-Alignment: 4194304
-Alignment: 8388608
-Alignment: 16777216
-Alignment: 33554432
-Test end

test/bitmap.c

@@ -1,157 +0,0 @@
-#define	JEMALLOC_MANGLE
-#include "jemalloc_test.h"
-
-/*
- * Avoid using the assert() from jemalloc_internal.h, since it requires
- * internal libjemalloc functionality.
- * */
-#include <assert.h>
-
-/*
- * Directly include the bitmap code, since it isn't exposed outside
- * libjemalloc.
- */
-#include "../src/bitmap.c"
-
-#if (LG_BITMAP_MAXBITS > 12)
-#  define MAXBITS	4500
-#else
-#  define MAXBITS	(1U << LG_BITMAP_MAXBITS)
-#endif
-
-static void
-test_bitmap_size(void)
-{
-	size_t i, prev_size;
-
-	prev_size = 0;
-	for (i = 1; i <= MAXBITS; i++) {
-		size_t size = bitmap_size(i);
-		assert(size >= prev_size);
-		prev_size = size;
-	}
-}
-
-static void
-test_bitmap_init(void)
-{
-	size_t i;
-
-	for (i = 1; i <= MAXBITS; i++) {
-		bitmap_info_t binfo;
-		bitmap_info_init(&binfo, i);
-		{
-			size_t j;
-			bitmap_t bitmap[bitmap_info_ngroups(&binfo)];
-			bitmap_init(bitmap, &binfo);
-
-			for (j = 0; j < i; j++)
-				assert(bitmap_get(bitmap, &binfo, j) == false);
-
-		}
-	}
-}
-
-static void
-test_bitmap_set(void)
-{
-	size_t i;
-
-	for (i = 1; i <= MAXBITS; i++) {
-		bitmap_info_t binfo;
-		bitmap_info_init(&binfo, i);
-		{
-			size_t j;
-			bitmap_t bitmap[bitmap_info_ngroups(&binfo)];
-			bitmap_init(bitmap, &binfo);
-
-			for (j = 0; j < i; j++)
-				bitmap_set(bitmap, &binfo, j);
-			assert(bitmap_full(bitmap, &binfo));
-		}
-	}
-}
-
-static void
-test_bitmap_unset(void)
-{
-	size_t i;
-
-	for (i = 1; i <= MAXBITS; i++) {
-		bitmap_info_t binfo;
-		bitmap_info_init(&binfo, i);
-		{
-			size_t j;
-			bitmap_t bitmap[bitmap_info_ngroups(&binfo)];
-			bitmap_init(bitmap, &binfo);
-
-			for (j = 0; j < i; j++)
-				bitmap_set(bitmap, &binfo, j);
-			assert(bitmap_full(bitmap, &binfo));
-			for (j = 0; j < i; j++)
-				bitmap_unset(bitmap, &binfo, j);
-			for (j = 0; j < i; j++)
-				bitmap_set(bitmap, &binfo, j);
-			assert(bitmap_full(bitmap, &binfo));
-		}
-	}
-}
-
-static void
-test_bitmap_sfu(void)
-{
-	size_t i;
-
-	for (i = 1; i <= MAXBITS; i++) {
-		bitmap_info_t binfo;
-		bitmap_info_init(&binfo, i);
-		{
-			ssize_t j;
-			bitmap_t bitmap[bitmap_info_ngroups(&binfo)];
-			bitmap_init(bitmap, &binfo);
-
-			/* Iteratively set bits starting at the beginning. */
-			for (j = 0; j < i; j++)
-				assert(bitmap_sfu(bitmap, &binfo) == j);
-			assert(bitmap_full(bitmap, &binfo));
-
-			/*
-			 * Iteratively unset bits starting at the end, and
-			 * verify that bitmap_sfu() reaches the unset bits.
-			 */
-			for (j = i - 1; j >= 0; j--) {
-				bitmap_unset(bitmap, &binfo, j);
-				assert(bitmap_sfu(bitmap, &binfo) == j);
-				bitmap_unset(bitmap, &binfo, j);
-			}
-			assert(bitmap_get(bitmap, &binfo, 0) == false);
-
-			/*
-			 * Iteratively set bits starting at the beginning, and
-			 * verify that bitmap_sfu() looks past them.
-			 */
-			for (j = 1; j < i; j++) {
-				bitmap_set(bitmap, &binfo, j - 1);
-				assert(bitmap_sfu(bitmap, &binfo) == j);
-				bitmap_unset(bitmap, &binfo, j);
-			}
-			assert(bitmap_sfu(bitmap, &binfo) == i - 1);
-			assert(bitmap_full(bitmap, &binfo));
-		}
-	}
-}
-
-int
-main(void)
-{
-	fprintf(stderr, "Test begin\n");
-
-	test_bitmap_size();
-	test_bitmap_init();
-	test_bitmap_set();
-	test_bitmap_unset();
-	test_bitmap_sfu();
-
-	fprintf(stderr, "Test end\n");
-	return (0);
-}

test/bitmap.exp

@@ -1,2 +0,0 @@
-Test begin
-Test end

test/include/test/SFMT-alti.h

@@ -0,0 +1,186 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/** 
+ * @file SFMT-alti.h 
+ *
+ * @brief SIMD oriented Fast Mersenne Twister(SFMT)
+ * pseudorandom number generator
+ *
+ * @author Mutsuo Saito (Hiroshima University)
+ * @author Makoto Matsumoto (Hiroshima University)
+ *
+ * Copyright (C) 2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ * University. All rights reserved.
+ *
+ * The new BSD License is applied to this software.
+ * see LICENSE.txt
+ */
+
+#ifndef SFMT_ALTI_H
+#define SFMT_ALTI_H
+
+/**
+ * This function represents the recursion formula in AltiVec and BIG ENDIAN.
+ * @param a a 128-bit part of the interal state array
+ * @param b a 128-bit part of the interal state array
+ * @param c a 128-bit part of the interal state array
+ * @param d a 128-bit part of the interal state array
+ * @return output
+ */
+JEMALLOC_ALWAYS_INLINE
+static vector unsigned int vec_recursion(vector unsigned int a,
+						vector unsigned int b,
+						vector unsigned int c,
+						vector unsigned int d) {
+
+    const vector unsigned int sl1 = ALTI_SL1;
+    const vector unsigned int sr1 = ALTI_SR1;
+#ifdef ONLY64
+    const vector unsigned int mask = ALTI_MSK64;
+    const vector unsigned char perm_sl = ALTI_SL2_PERM64;
+    const vector unsigned char perm_sr = ALTI_SR2_PERM64;
+#else
+    const vector unsigned int mask = ALTI_MSK;
+    const vector unsigned char perm_sl = ALTI_SL2_PERM;
+    const vector unsigned char perm_sr = ALTI_SR2_PERM;
+#endif
+    vector unsigned int v, w, x, y, z;
+    x = vec_perm(a, (vector unsigned int)perm_sl, perm_sl);
+    v = a;
+    y = vec_sr(b, sr1);
+    z = vec_perm(c, (vector unsigned int)perm_sr, perm_sr);
+    w = vec_sl(d, sl1);
+    z = vec_xor(z, w);
+    y = vec_and(y, mask);
+    v = vec_xor(v, x);
+    z = vec_xor(z, y);
+    z = vec_xor(z, v);
+    return z;
+}
+
+/**
+ * This function fills the internal state array with pseudorandom
+ * integers.
+ */
+JEMALLOC_INLINE void gen_rand_all(sfmt_t *ctx) {
+    int i;
+    vector unsigned int r, r1, r2;
+
+    r1 = ctx->sfmt[N - 2].s;
+    r2 = ctx->sfmt[N - 1].s;
+    for (i = 0; i < N - POS1; i++) {
+	r = vec_recursion(ctx->sfmt[i].s, ctx->sfmt[i + POS1].s, r1, r2);
+	ctx->sfmt[i].s = r;
+	r1 = r2;
+	r2 = r;
+    }
+    for (; i < N; i++) {
+	r = vec_recursion(ctx->sfmt[i].s, ctx->sfmt[i + POS1 - N].s, r1, r2);
+	ctx->sfmt[i].s = r;
+	r1 = r2;
+	r2 = r;
+    }
+}
+
+/**
+ * This function fills the user-specified array with pseudorandom
+ * integers.
+ *
+ * @param array an 128-bit array to be filled by pseudorandom numbers.  
+ * @param size number of 128-bit pesudorandom numbers to be generated.
+ */
+JEMALLOC_INLINE void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) {
+    int i, j;
+    vector unsigned int r, r1, r2;
+
+    r1 = ctx->sfmt[N - 2].s;
+    r2 = ctx->sfmt[N - 1].s;
+    for (i = 0; i < N - POS1; i++) {
+	r = vec_recursion(ctx->sfmt[i].s, ctx->sfmt[i + POS1].s, r1, r2);
+	array[i].s = r;
+	r1 = r2;
+	r2 = r;
+    }
+    for (; i < N; i++) {
+	r = vec_recursion(ctx->sfmt[i].s, array[i + POS1 - N].s, r1, r2);
+	array[i].s = r;
+	r1 = r2;
+	r2 = r;
+    }
+    /* main loop */
+    for (; i < size - N; i++) {
+	r = vec_recursion(array[i - N].s, array[i + POS1 - N].s, r1, r2);
+	array[i].s = r;
+	r1 = r2;
+	r2 = r;
+    }
+    for (j = 0; j < 2 * N - size; j++) {
+	ctx->sfmt[j].s = array[j + size - N].s;
+    }
+    for (; i < size; i++) {
+	r = vec_recursion(array[i - N].s, array[i + POS1 - N].s, r1, r2);
+	array[i].s = r;
+	ctx->sfmt[j++].s = r;
+	r1 = r2;
+	r2 = r;
+    }
+}
+
+#ifndef ONLY64
+#if defined(__APPLE__)
+#define ALTI_SWAP (vector unsigned char) \
+	(4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11)
+#else
+#define ALTI_SWAP {4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11}
+#endif
+/**
+ * This function swaps high and low 32-bit of 64-bit integers in user
+ * specified array.
+ *
+ * @param array an 128-bit array to be swaped.
+ * @param size size of 128-bit array.
+ */
+JEMALLOC_INLINE void swap(w128_t *array, int size) {
+    int i;
+    const vector unsigned char perm = ALTI_SWAP;
+
+    for (i = 0; i < size; i++) {
+	array[i].s = vec_perm(array[i].s, (vector unsigned int)perm, perm);
+    }
+}
+#endif
+
+#endif

test/include/test/SFMT-params.h

@@ -0,0 +1,132 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS_H
+#define SFMT_PARAMS_H
+
+#if !defined(MEXP)
+#ifdef __GNUC__
+  #warning "MEXP is not defined. I assume MEXP is 19937."
+#endif
+  #define MEXP 19937
+#endif
+/*-----------------
+  BASIC DEFINITIONS
+  -----------------*/
+/** Mersenne Exponent. The period of the sequence 
+ *  is a multiple of 2^MEXP-1.
+ * #define MEXP 19937 */
+/** SFMT generator has an internal state array of 128-bit integers,
+ * and N is its size. */
+#define N (MEXP / 128 + 1)
+/** N32 is the size of internal state array when regarded as an array
+ * of 32-bit integers.*/
+#define N32 (N * 4)
+/** N64 is the size of internal state array when regarded as an array
+ * of 64-bit integers.*/
+#define N64 (N * 2)
+
+/*----------------------
+  the parameters of SFMT
+  following definitions are in paramsXXXX.h file.
+  ----------------------*/
+/** the pick up position of the array.
+#define POS1 122 
+*/
+
+/** the parameter of shift left as four 32-bit registers.
+#define SL1 18
+ */
+
+/** the parameter of shift left as one 128-bit register. 
+ * The 128-bit integer is shifted by (SL2 * 8) bits. 
+#define SL2 1 
+*/
+
+/** the parameter of shift right as four 32-bit registers.
+#define SR1 11
+*/
+
+/** the parameter of shift right as one 128-bit register. 
+ * The 128-bit integer is shifted by (SL2 * 8) bits. 
+#define SR2 1 
+*/
+
+/** A bitmask, used in the recursion.  These parameters are introduced
+ * to break symmetry of SIMD.
+#define MSK1 0xdfffffefU
+#define MSK2 0xddfecb7fU
+#define MSK3 0xbffaffffU
+#define MSK4 0xbffffff6U 
+*/
+
+/** These definitions are part of a 128-bit period certification vector.
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0x00000000U
+#define PARITY4	0xc98e126aU
+*/
+
+#if MEXP == 607
+  #include "test/SFMT-params607.h"
+#elif MEXP == 1279
+  #include "test/SFMT-params1279.h"
+#elif MEXP == 2281
+  #include "test/SFMT-params2281.h"
+#elif MEXP == 4253
+  #include "test/SFMT-params4253.h"
+#elif MEXP == 11213
+  #include "test/SFMT-params11213.h"
+#elif MEXP == 19937
+  #include "test/SFMT-params19937.h"
+#elif MEXP == 44497
+  #include "test/SFMT-params44497.h"
+#elif MEXP == 86243
+  #include "test/SFMT-params86243.h"
+#elif MEXP == 132049
+  #include "test/SFMT-params132049.h"
+#elif MEXP == 216091
+  #include "test/SFMT-params216091.h"
+#else
+#ifdef __GNUC__
+  #error "MEXP is not valid."
+  #undef MEXP
+#else
+  #undef MEXP
+#endif
+
+#endif
+
+#endif /* SFMT_PARAMS_H */

test/include/test/SFMT-params11213.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS11213_H
+#define SFMT_PARAMS11213_H
+
+#define POS1	68
+#define SL1	14
+#define SL2	3
+#define SR1	7
+#define SR2	3
+#define MSK1	0xeffff7fbU
+#define MSK2	0xffffffefU
+#define MSK3	0xdfdfbfffU
+#define MSK4	0x7fffdbfdU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0xe8148000U
+#define PARITY4	0xd0c7afa3U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10}
+    #define ALTI_SL2_PERM64	{3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2}
+    #define ALTI_SR2_PERM	{5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12}
+    #define ALTI_SR2_PERM64	{13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-11213:68-14-3-7-3:effff7fb-ffffffef-dfdfbfff-7fffdbfd"
+
+#endif /* SFMT_PARAMS11213_H */

test/include/test/SFMT-params1279.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS1279_H
+#define SFMT_PARAMS1279_H
+
+#define POS1	7
+#define SL1	14
+#define SL2	3
+#define SR1	5
+#define SR2	1
+#define MSK1	0xf7fefffdU
+#define MSK2	0x7fefcfffU
+#define MSK3	0xaff3ef3fU
+#define MSK4	0xb5ffff7fU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0x00000000U
+#define PARITY4	0x20000000U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10}
+    #define ALTI_SL2_PERM64	{3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-1279:7-14-3-5-1:f7fefffd-7fefcfff-aff3ef3f-b5ffff7f"
+
+#endif /* SFMT_PARAMS1279_H */

test/include/test/SFMT-params132049.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS132049_H
+#define SFMT_PARAMS132049_H
+
+#define POS1	110
+#define SL1	19
+#define SL2	1
+#define SR1	21
+#define SR2	1
+#define MSK1	0xffffbb5fU
+#define MSK2	0xfb6ebf95U
+#define MSK3	0xfffefffaU
+#define MSK4	0xcff77fffU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0xcb520000U
+#define PARITY4	0xc7e91c7dU
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8}
+    #define ALTI_SL2_PERM64	{1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-132049:110-19-1-21-1:ffffbb5f-fb6ebf95-fffefffa-cff77fff"
+
+#endif /* SFMT_PARAMS132049_H */

test/include/test/SFMT-params19937.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS19937_H
+#define SFMT_PARAMS19937_H
+
+#define POS1	122
+#define SL1	18
+#define SL2	1
+#define SR1	11
+#define SR2	1
+#define MSK1	0xdfffffefU
+#define MSK2	0xddfecb7fU
+#define MSK3	0xbffaffffU
+#define MSK4	0xbffffff6U
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0x00000000U
+#define PARITY4	0x13c9e684U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8}
+    #define ALTI_SL2_PERM64	{1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-19937:122-18-1-11-1:dfffffef-ddfecb7f-bffaffff-bffffff6"
+
+#endif /* SFMT_PARAMS19937_H */

test/include/test/SFMT-params216091.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS216091_H
+#define SFMT_PARAMS216091_H
+
+#define POS1	627
+#define SL1	11
+#define SL2	3
+#define SR1	10
+#define SR2	1
+#define MSK1	0xbff7bff7U
+#define MSK2	0xbfffffffU
+#define MSK3	0xbffffa7fU
+#define MSK4	0xffddfbfbU
+#define PARITY1	0xf8000001U
+#define PARITY2	0x89e80709U
+#define PARITY3	0x3bd2b64bU
+#define PARITY4	0x0c64b1e4U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10}
+    #define ALTI_SL2_PERM64	{3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-216091:627-11-3-10-1:bff7bff7-bfffffff-bffffa7f-ffddfbfb"
+
+#endif /* SFMT_PARAMS216091_H */

test/include/test/SFMT-params2281.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS2281_H
+#define SFMT_PARAMS2281_H
+
+#define POS1	12
+#define SL1	19
+#define SL2	1
+#define SR1	5
+#define SR2	1
+#define MSK1	0xbff7ffbfU
+#define MSK2	0xfdfffffeU
+#define MSK3	0xf7ffef7fU
+#define MSK4	0xf2f7cbbfU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0x00000000U
+#define PARITY4	0x41dfa600U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8}
+    #define ALTI_SL2_PERM64	{1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-2281:12-19-1-5-1:bff7ffbf-fdfffffe-f7ffef7f-f2f7cbbf"
+
+#endif /* SFMT_PARAMS2281_H */

test/include/test/SFMT-params4253.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS4253_H
+#define SFMT_PARAMS4253_H
+
+#define POS1	17
+#define SL1	20
+#define SL2	1
+#define SR1	7
+#define SR2	1
+#define MSK1	0x9f7bffffU
+#define MSK2	0x9fffff5fU
+#define MSK3	0x3efffffbU
+#define MSK4	0xfffff7bbU
+#define PARITY1	0xa8000001U
+#define PARITY2	0xaf5390a3U
+#define PARITY3	0xb740b3f8U
+#define PARITY4	0x6c11486dU
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8}
+    #define ALTI_SL2_PERM64	{1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-4253:17-20-1-7-1:9f7bffff-9fffff5f-3efffffb-fffff7bb"
+
+#endif /* SFMT_PARAMS4253_H */

test/include/test/SFMT-params44497.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS44497_H
+#define SFMT_PARAMS44497_H
+
+#define POS1	330
+#define SL1	5
+#define SL2	3
+#define SR1	9
+#define SR2	3
+#define MSK1	0xeffffffbU
+#define MSK2	0xdfbebfffU
+#define MSK3	0xbfbf7befU
+#define MSK4	0x9ffd7bffU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0xa3ac4000U
+#define PARITY4	0xecc1327aU
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10}
+    #define ALTI_SL2_PERM64	{3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2}
+    #define ALTI_SR2_PERM	{5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12}
+    #define ALTI_SR2_PERM64	{13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-44497:330-5-3-9-3:effffffb-dfbebfff-bfbf7bef-9ffd7bff"
+
+#endif /* SFMT_PARAMS44497_H */

test/include/test/SFMT-params607.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS607_H
+#define SFMT_PARAMS607_H
+
+#define POS1	2
+#define SL1	15
+#define SL2	3
+#define SR1	13
+#define SR2	3
+#define MSK1	0xfdff37ffU
+#define MSK2	0xef7f3f7dU
+#define MSK3	0xff777b7dU
+#define MSK4	0x7ff7fb2fU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0x00000000U
+#define PARITY4	0x5986f054U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10}
+    #define ALTI_SL2_PERM64	{3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2}
+    #define ALTI_SR2_PERM	{5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12}
+    #define ALTI_SR2_PERM64	{13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-607:2-15-3-13-3:fdff37ff-ef7f3f7d-ff777b7d-7ff7fb2f"
+
+#endif /* SFMT_PARAMS607_H */

test/include/test/SFMT-params86243.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS86243_H
+#define SFMT_PARAMS86243_H
+
+#define POS1	366
+#define SL1	6
+#define SL2	7
+#define SR1	19
+#define SR2	1
+#define MSK1	0xfdbffbffU
+#define MSK2	0xbff7ff3fU
+#define MSK3	0xfd77efffU
+#define MSK4	0xbf9ff3ffU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0x00000000U
+#define PARITY4	0xe9528d85U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(25,25,25,25,3,25,25,25,7,0,1,2,11,4,5,6)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(7,25,25,25,25,25,25,25,15,0,1,2,3,4,5,6)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{25,25,25,25,3,25,25,25,7,0,1,2,11,4,5,6}
+    #define ALTI_SL2_PERM64	{7,25,25,25,25,25,25,25,15,0,1,2,3,4,5,6}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-86243:366-6-7-19-1:fdbffbff-bff7ff3f-fd77efff-bf9ff3ff"
+
+#endif /* SFMT_PARAMS86243_H */

test/include/test/SFMT-sse2.h

@@ -0,0 +1,157 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/** 
+ * @file  SFMT-sse2.h
+ * @brief SIMD oriented Fast Mersenne Twister(SFMT) for Intel SSE2
+ *
+ * @author Mutsuo Saito (Hiroshima University)
+ * @author Makoto Matsumoto (Hiroshima University)
+ *
+ * @note We assume LITTLE ENDIAN in this file
+ *
+ * Copyright (C) 2006, 2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ * University. All rights reserved.
+ *
+ * The new BSD License is applied to this software, see LICENSE.txt
+ */
+
+#ifndef SFMT_SSE2_H
+#define SFMT_SSE2_H
+
+/**
+ * This function represents the recursion formula.
+ * @param a a 128-bit part of the interal state array
+ * @param b a 128-bit part of the interal state array
+ * @param c a 128-bit part of the interal state array
+ * @param d a 128-bit part of the interal state array
+ * @param mask 128-bit mask
+ * @return output
+ */
+JEMALLOC_ALWAYS_INLINE __m128i mm_recursion(__m128i *a, __m128i *b, 
+				   __m128i c, __m128i d, __m128i mask) {
+    __m128i v, x, y, z;
+    
+    x = _mm_load_si128(a);
+    y = _mm_srli_epi32(*b, SR1);
+    z = _mm_srli_si128(c, SR2);
+    v = _mm_slli_epi32(d, SL1);
+    z = _mm_xor_si128(z, x);
+    z = _mm_xor_si128(z, v);
+    x = _mm_slli_si128(x, SL2);
+    y = _mm_and_si128(y, mask);
+    z = _mm_xor_si128(z, x);
+    z = _mm_xor_si128(z, y);
+    return z;
+}
+
+/**
+ * This function fills the internal state array with pseudorandom
+ * integers.
+ */
+JEMALLOC_INLINE void gen_rand_all(sfmt_t *ctx) {
+    int i;
+    __m128i r, r1, r2, mask;
+    mask = _mm_set_epi32(MSK4, MSK3, MSK2, MSK1);
+
+    r1 = _mm_load_si128(&ctx->sfmt[N - 2].si);
+    r2 = _mm_load_si128(&ctx->sfmt[N - 1].si);
+    for (i = 0; i < N - POS1; i++) {
+	r = mm_recursion(&ctx->sfmt[i].si, &ctx->sfmt[i + POS1].si, r1, r2,
+	  mask);
+	_mm_store_si128(&ctx->sfmt[i].si, r);
+	r1 = r2;
+	r2 = r;
+    }
+    for (; i < N; i++) {
+	r = mm_recursion(&ctx->sfmt[i].si, &ctx->sfmt[i + POS1 - N].si, r1, r2,
+	  mask);
+	_mm_store_si128(&ctx->sfmt[i].si, r);
+	r1 = r2;
+	r2 = r;
+    }
+}
+
+/**
+ * This function fills the user-specified array with pseudorandom
+ * integers.
+ *
+ * @param array an 128-bit array to be filled by pseudorandom numbers.  
+ * @param size number of 128-bit pesudorandom numbers to be generated.
+ */
+JEMALLOC_INLINE void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) {
+    int i, j;
+    __m128i r, r1, r2, mask;
+    mask = _mm_set_epi32(MSK4, MSK3, MSK2, MSK1);
+
+    r1 = _mm_load_si128(&ctx->sfmt[N - 2].si);
+    r2 = _mm_load_si128(&ctx->sfmt[N - 1].si);
+    for (i = 0; i < N - POS1; i++) {
+	r = mm_recursion(&ctx->sfmt[i].si, &ctx->sfmt[i + POS1].si, r1, r2,
+	  mask);
+	_mm_store_si128(&array[i].si, r);
+	r1 = r2;
+	r2 = r;
+    }
+    for (; i < N; i++) {
+	r = mm_recursion(&ctx->sfmt[i].si, &array[i + POS1 - N].si, r1, r2,
+	  mask);
+	_mm_store_si128(&array[i].si, r);
+	r1 = r2;
+	r2 = r;
+    }
+    /* main loop */
+    for (; i < size - N; i++) {
+	r = mm_recursion(&array[i - N].si, &array[i + POS1 - N].si, r1, r2,
+			 mask);
+	_mm_store_si128(&array[i].si, r);
+	r1 = r2;
+	r2 = r;
+    }
+    for (j = 0; j < 2 * N - size; j++) {
+	r = _mm_load_si128(&array[j + size - N].si);
+	_mm_store_si128(&ctx->sfmt[j].si, r);
+    }
+    for (; i < size; i++) {
+	r = mm_recursion(&array[i - N].si, &array[i + POS1 - N].si, r1, r2,
+			 mask);
+	_mm_store_si128(&array[i].si, r);
+	_mm_store_si128(&ctx->sfmt[j++].si, r);
+	r1 = r2;
+	r2 = r;
+    }
+}
+
+#endif