mirror of
https://github.com/jemalloc/jemalloc.git
synced 2026-07-25 06:13:13 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
1819ae581c | ||
|
|
1ba886fb69 |
@@ -1,28 +0,0 @@
|
||||
version: '{build}'
|
||||
|
||||
environment:
|
||||
matrix:
|
||||
- MSYSTEM: MINGW64
|
||||
CPU: x86_64
|
||||
MSVC: amd64
|
||||
- MSYSTEM: MINGW32
|
||||
CPU: i686
|
||||
MSVC: x86
|
||||
- MSYSTEM: MINGW64
|
||||
CPU: x86_64
|
||||
- MSYSTEM: MINGW32
|
||||
CPU: i686
|
||||
|
||||
install:
|
||||
- set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH%
|
||||
- if defined MSVC call "c:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %MSVC%
|
||||
- if defined MSVC pacman --noconfirm -Rsc mingw-w64-%CPU%-gcc gcc
|
||||
- pacman --noconfirm -Suy mingw-w64-%CPU%-make
|
||||
|
||||
build_script:
|
||||
- bash -c "autoconf"
|
||||
- bash -c "./configure"
|
||||
- mingw32-make -j3
|
||||
- file lib/jemalloc.dll
|
||||
- mingw32-make -j3 tests
|
||||
- mingw32-make -k check
|
||||
@@ -1,3 +0,0 @@
|
||||
begin-language: "Autoconf-without-aclocal-m4"
|
||||
args: --no-cache
|
||||
end-language: "Autoconf-without-aclocal-m4"
|
||||
1
.gitattributes
vendored
1
.gitattributes
vendored
@@ -1 +0,0 @@
|
||||
* text=auto eol=lf
|
||||
23
.gitignore
vendored
23
.gitignore
vendored
@@ -1,8 +1,8 @@
|
||||
/*.gcov.*
|
||||
|
||||
/bin/jemalloc-config
|
||||
/autom4te.cache/
|
||||
|
||||
/bin/jemalloc.sh
|
||||
/bin/jeprof
|
||||
|
||||
/config.stamp
|
||||
/config.log
|
||||
@@ -15,8 +15,6 @@
|
||||
/doc/jemalloc.html
|
||||
/doc/jemalloc.3
|
||||
|
||||
/jemalloc.pc
|
||||
|
||||
/lib/
|
||||
|
||||
/Makefile
|
||||
@@ -37,7 +35,6 @@
|
||||
/include/jemalloc/jemalloc_protos.h
|
||||
/include/jemalloc/jemalloc_protos_jet.h
|
||||
/include/jemalloc/jemalloc_rename.h
|
||||
/include/jemalloc/jemalloc_typedefs.h
|
||||
|
||||
/src/*.[od]
|
||||
/src/*.gcda
|
||||
@@ -73,19 +70,3 @@ test/include/test/jemalloc_test_defs.h
|
||||
/test/unit/*.out
|
||||
|
||||
/VERSION
|
||||
|
||||
*.pdb
|
||||
*.sdf
|
||||
*.opendb
|
||||
*.opensdf
|
||||
*.cachefile
|
||||
*.suo
|
||||
*.user
|
||||
*.sln.docstates
|
||||
*.tmp
|
||||
/msvc/Win32/
|
||||
/msvc/x64/
|
||||
/msvc/projects/*/*/Debug*/
|
||||
/msvc/projects/*/*/Release*/
|
||||
/msvc/projects/*/*/Win32/
|
||||
/msvc/projects/*/*/x64/
|
||||
|
||||
29
.travis.yml
29
.travis.yml
@@ -1,29 +0,0 @@
|
||||
language: c
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- os: linux
|
||||
compiler: gcc
|
||||
- os: linux
|
||||
compiler: gcc
|
||||
env:
|
||||
- EXTRA_FLAGS=-m32
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- gcc-multilib
|
||||
- os: osx
|
||||
compiler: clang
|
||||
- os: osx
|
||||
compiler: clang
|
||||
env:
|
||||
- EXTRA_FLAGS=-m32
|
||||
|
||||
before_script:
|
||||
- autoconf
|
||||
- ./configure${EXTRA_FLAGS:+ CC="$CC $EXTRA_FLAGS"}
|
||||
- make -j3
|
||||
- make -j3 tests
|
||||
|
||||
script:
|
||||
- make check
|
||||
4
COPYING
4
COPYING
@@ -1,10 +1,10 @@
|
||||
Unless otherwise specified, files in the jemalloc source distribution are
|
||||
subject to the following license:
|
||||
--------------------------------------------------------------------------------
|
||||
Copyright (C) 2002-2016 Jason Evans <jasone@canonware.com>.
|
||||
Copyright (C) 2002-2014 Jason Evans <jasone@canonware.com>.
|
||||
All rights reserved.
|
||||
Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved.
|
||||
Copyright (C) 2009-2016 Facebook, Inc. All rights reserved.
|
||||
Copyright (C) 2009-2014 Facebook, Inc. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
470
ChangeLog
470
ChangeLog
@@ -1,470 +1,10 @@
|
||||
Following are change highlights associated with official releases. Important
|
||||
bug fixes are all mentioned, but some internal enhancements are omitted here for
|
||||
brevity. Much more detail can be found in the git revision history:
|
||||
bug fixes are all mentioned, but internal enhancements are omitted here for
|
||||
brevity (even though they are more fun to write about). Much more detail can be
|
||||
found in the git revision history:
|
||||
|
||||
https://github.com/jemalloc/jemalloc
|
||||
|
||||
* 4.4.0 (December 3, 2016)
|
||||
|
||||
New features:
|
||||
- Add configure support for *-*-linux-android. (@cferris1000, @jasone)
|
||||
- Add the --disable-syscall configure option, for use on systems that place
|
||||
security-motivated limitations on syscall(2). (@jasone)
|
||||
- Add support for Debian GNU/kFreeBSD. (@thesam)
|
||||
|
||||
Optimizations:
|
||||
- Add extent serial numbers and use them where appropriate as a sort key that
|
||||
is higher priority than address, so that the allocation policy prefers older
|
||||
extents. This tends to improve locality (decrease fragmentation) when
|
||||
memory grows downward. (@jasone)
|
||||
- Refactor madvise(2) configuration so that MADV_FREE is detected and utilized
|
||||
on Linux 4.5 and newer. (@jasone)
|
||||
- Mark partially purged arena chunks as non-huge-page. This improves
|
||||
interaction with Linux's transparent huge page functionality. (@jasone)
|
||||
|
||||
Bug fixes:
|
||||
- Fix size class computations for edge conditions involving extremely large
|
||||
allocations. This regression was first released in 4.0.0. (@jasone,
|
||||
@ingvarha)
|
||||
- Remove overly restrictive assertions related to the cactive statistic. This
|
||||
regression was first released in 4.1.0. (@jasone)
|
||||
- Implement a more reliable detection scheme for os_unfair_lock on macOS.
|
||||
(@jszakmeister)
|
||||
|
||||
* 4.3.1 (November 7, 2016)
|
||||
|
||||
Bug fixes:
|
||||
- Fix a severe virtual memory leak. This regression was first released in
|
||||
4.3.0. (@interwq, @jasone)
|
||||
- Refactor atomic and prng APIs to restore support for 32-bit platforms that
|
||||
use pre-C11 toolchains, e.g. FreeBSD's mips. (@jasone)
|
||||
|
||||
* 4.3.0 (November 4, 2016)
|
||||
|
||||
This is the first release that passes the test suite for multiple Windows
|
||||
configurations, thanks in large part to @glandium setting up continuous
|
||||
integration via AppVeyor (and Travis CI for Linux and OS X).
|
||||
|
||||
New features:
|
||||
- Add "J" (JSON) support to malloc_stats_print(). (@jasone)
|
||||
- Add Cray compiler support. (@ronawho)
|
||||
|
||||
Optimizations:
|
||||
- Add/use adaptive spinning for bootstrapping and radix tree node
|
||||
initialization. (@jasone)
|
||||
|
||||
Bug fixes:
|
||||
- Fix large allocation to search starting in the optimal size class heap,
|
||||
which can substantially reduce virtual memory churn and fragmentation. This
|
||||
regression was first released in 4.0.0. (@mjp41, @jasone)
|
||||
- Fix stats.arenas.<i>.nthreads accounting. (@interwq)
|
||||
- Fix and simplify decay-based purging. (@jasone)
|
||||
- Make DSS (sbrk(2)-related) operations lockless, which resolves potential
|
||||
deadlocks during thread exit. (@jasone)
|
||||
- Fix over-sized allocation of radix tree leaf nodes. (@mjp41, @ogaun,
|
||||
@jasone)
|
||||
- Fix over-sized allocation of arena_t (plus associated stats) data
|
||||
structures. (@jasone, @interwq)
|
||||
- Fix EXTRA_CFLAGS to not affect configuration. (@jasone)
|
||||
- Fix a Valgrind integration bug. (@ronawho)
|
||||
- Disallow 0x5a junk filling when running in Valgrind. (@jasone)
|
||||
- Fix a file descriptor leak on Linux. This regression was first released in
|
||||
4.2.0. (@vsarunas, @jasone)
|
||||
- Fix static linking of jemalloc with glibc. (@djwatson)
|
||||
- Use syscall(2) rather than {open,read,close}(2) during boot on Linux. This
|
||||
works around other libraries' system call wrappers performing reentrant
|
||||
allocation. (@kspinka, @Whissi, @jasone)
|
||||
- Fix OS X default zone replacement to work with OS X 10.12. (@glandium,
|
||||
@jasone)
|
||||
- Fix cached memory management to avoid needless commit/decommit operations
|
||||
during purging, which resolves permanent virtual memory map fragmentation
|
||||
issues on Windows. (@mjp41, @jasone)
|
||||
- Fix TSD fetches to avoid (recursive) allocation. This is relevant to
|
||||
non-TLS and Windows configurations. (@jasone)
|
||||
- Fix malloc_conf overriding to work on Windows. (@jasone)
|
||||
- Forcibly disable lazy-lock on Windows (was forcibly *enabled*). (@jasone)
|
||||
|
||||
* 4.2.1 (June 8, 2016)
|
||||
|
||||
Bug fixes:
|
||||
- Fix bootstrapping issues for configurations that require allocation during
|
||||
tsd initialization (e.g. --disable-tls). (@cferris1000, @jasone)
|
||||
- Fix gettimeofday() version of nstime_update(). (@ronawho)
|
||||
- Fix Valgrind regressions in calloc() and chunk_alloc_wrapper(). (@ronawho)
|
||||
- Fix potential VM map fragmentation regression. (@jasone)
|
||||
- Fix opt_zero-triggered in-place huge reallocation zeroing. (@jasone)
|
||||
- Fix heap profiling context leaks in reallocation edge cases. (@jasone)
|
||||
|
||||
* 4.2.0 (May 12, 2016)
|
||||
|
||||
New features:
|
||||
- Add the arena.<i>.reset mallctl, which makes it possible to discard all of
|
||||
an arena's allocations in a single operation. (@jasone)
|
||||
- Add the stats.retained and stats.arenas.<i>.retained statistics. (@jasone)
|
||||
- Add the --with-version configure option. (@jasone)
|
||||
- Support --with-lg-page values larger than actual page size. (@jasone)
|
||||
|
||||
Optimizations:
|
||||
- Use pairing heaps rather than red-black trees for various hot data
|
||||
structures. (@djwatson, @jasone)
|
||||
- Streamline fast paths of rtree operations. (@jasone)
|
||||
- Optimize the fast paths of calloc() and [m,d,sd]allocx(). (@jasone)
|
||||
- Decommit unused virtual memory if the OS does not overcommit. (@jasone)
|
||||
- Specify MAP_NORESERVE on Linux if [heuristic] overcommit is active, in order
|
||||
to avoid unfortunate interactions during fork(2). (@jasone)
|
||||
|
||||
Bug fixes:
|
||||
- Fix chunk accounting related to triggering gdump profiles. (@jasone)
|
||||
- Link against librt for clock_gettime(2) if glibc < 2.17. (@jasone)
|
||||
- Scale leak report summary according to sampling probability. (@jasone)
|
||||
|
||||
* 4.1.1 (May 3, 2016)
|
||||
|
||||
This bugfix release resolves a variety of mostly minor issues, though the
|
||||
bitmap fix is critical for 64-bit Windows.
|
||||
|
||||
Bug fixes:
|
||||
- Fix the linear scan version of bitmap_sfu() to shift by the proper amount
|
||||
even when sizeof(long) is not the same as sizeof(void *), as on 64-bit
|
||||
Windows. (@jasone)
|
||||
- Fix hashing functions to avoid unaligned memory accesses (and resulting
|
||||
crashes). This is relevant at least to some ARM-based platforms.
|
||||
(@rkmisra)
|
||||
- Fix fork()-related lock rank ordering reversals. These reversals were
|
||||
unlikely to cause deadlocks in practice except when heap profiling was
|
||||
enabled and active. (@jasone)
|
||||
- Fix various chunk leaks in OOM code paths. (@jasone)
|
||||
- Fix malloc_stats_print() to print opt.narenas correctly. (@jasone)
|
||||
- Fix MSVC-specific build/test issues. (@rustyx, @yuslepukhin)
|
||||
- Fix a variety of test failures that were due to test fragility rather than
|
||||
core bugs. (@jasone)
|
||||
|
||||
* 4.1.0 (February 28, 2016)
|
||||
|
||||
This release is primarily about optimizations, but it also incorporates a lot
|
||||
of portability-motivated refactoring and enhancements. Many people worked on
|
||||
this release, to an extent that even with the omission here of minor changes
|
||||
(see git revision history), and of the people who reported and diagnosed
|
||||
issues, so much of the work was contributed that starting with this release,
|
||||
changes are annotated with author credits to help reflect the collaborative
|
||||
effort involved.
|
||||
|
||||
New features:
|
||||
- Implement decay-based unused dirty page purging, a major optimization with
|
||||
mallctl API impact. This is an alternative to the existing ratio-based
|
||||
unused dirty page purging, and is intended to eventually become the sole
|
||||
purging mechanism. New mallctls:
|
||||
+ opt.purge
|
||||
+ opt.decay_time
|
||||
+ arena.<i>.decay
|
||||
+ arena.<i>.decay_time
|
||||
+ arenas.decay_time
|
||||
+ stats.arenas.<i>.decay_time
|
||||
(@jasone, @cevans87)
|
||||
- Add --with-malloc-conf, which makes it possible to embed a default
|
||||
options string during configuration. This was motivated by the desire to
|
||||
specify --with-malloc-conf=purge:decay , since the default must remain
|
||||
purge:ratio until the 5.0.0 release. (@jasone)
|
||||
- Add MS Visual Studio 2015 support. (@rustyx, @yuslepukhin)
|
||||
- Make *allocx() size class overflow behavior defined. The maximum
|
||||
size class is now less than PTRDIFF_MAX to protect applications against
|
||||
numerical overflow, and all allocation functions are guaranteed to indicate
|
||||
errors rather than potentially crashing if the request size exceeds the
|
||||
maximum size class. (@jasone)
|
||||
- jeprof:
|
||||
+ Add raw heap profile support. (@jasone)
|
||||
+ Add --retain and --exclude for backtrace symbol filtering. (@jasone)
|
||||
|
||||
Optimizations:
|
||||
- Optimize the fast path to combine various bootstrapping and configuration
|
||||
checks and execute more streamlined code in the common case. (@interwq)
|
||||
- Use linear scan for small bitmaps (used for small object tracking). In
|
||||
addition to speeding up bitmap operations on 64-bit systems, this reduces
|
||||
allocator metadata overhead by approximately 0.2%. (@djwatson)
|
||||
- Separate arena_avail trees, which substantially speeds up run tree
|
||||
operations. (@djwatson)
|
||||
- Use memoization (boot-time-computed table) for run quantization. Separate
|
||||
arena_avail trees reduced the importance of this optimization. (@jasone)
|
||||
- Attempt mmap-based in-place huge reallocation. This can dramatically speed
|
||||
up incremental huge reallocation. (@jasone)
|
||||
|
||||
Incompatible changes:
|
||||
- Make opt.narenas unsigned rather than size_t. (@jasone)
|
||||
|
||||
Bug fixes:
|
||||
- Fix stats.cactive accounting regression. (@rustyx, @jasone)
|
||||
- Handle unaligned keys in hash(). This caused problems for some ARM systems.
|
||||
(@jasone, @cferris1000)
|
||||
- Refactor arenas array. In addition to fixing a fork-related deadlock, this
|
||||
makes arena lookups faster and simpler. (@jasone)
|
||||
- Move retained memory allocation out of the default chunk allocation
|
||||
function, to a location that gets executed even if the application installs
|
||||
a custom chunk allocation function. This resolves a virtual memory leak.
|
||||
(@buchgr)
|
||||
- Fix a potential tsd cleanup leak. (@cferris1000, @jasone)
|
||||
- Fix run quantization. In practice this bug had no impact unless
|
||||
applications requested memory with alignment exceeding one page.
|
||||
(@jasone, @djwatson)
|
||||
- Fix LinuxThreads-specific bootstrapping deadlock. (Cosmin Paraschiv)
|
||||
- jeprof:
|
||||
+ Don't discard curl options if timeout is not defined. (@djwatson)
|
||||
+ Detect failed profile fetches. (@djwatson)
|
||||
- Fix stats.arenas.<i>.{dss,lg_dirty_mult,decay_time,pactive,pdirty} for
|
||||
--disable-stats case. (@jasone)
|
||||
|
||||
* 4.0.4 (October 24, 2015)
|
||||
|
||||
This bugfix release fixes another xallocx() regression. No other regressions
|
||||
have come to light in over a month, so this is likely a good starting point
|
||||
for people who prefer to wait for "dot one" releases with all the major issues
|
||||
shaken out.
|
||||
|
||||
Bug fixes:
|
||||
- Fix xallocx(..., MALLOCX_ZERO to zero the last full trailing page of large
|
||||
allocations that have been randomly assigned an offset of 0 when
|
||||
--enable-cache-oblivious configure option is enabled.
|
||||
|
||||
* 4.0.3 (September 24, 2015)
|
||||
|
||||
This bugfix release continues the trend of xallocx() and heap profiling fixes.
|
||||
|
||||
Bug fixes:
|
||||
- Fix xallocx(..., MALLOCX_ZERO) to zero all trailing bytes of large
|
||||
allocations when --enable-cache-oblivious configure option is enabled.
|
||||
- Fix xallocx(..., MALLOCX_ZERO) to zero trailing bytes of huge allocations
|
||||
when resizing from/to a size class that is not a multiple of the chunk size.
|
||||
- Fix prof_tctx_dump_iter() to filter out nodes that were created after heap
|
||||
profile dumping started.
|
||||
- Work around a potentially bad thread-specific data initialization
|
||||
interaction with NPTL (glibc's pthreads implementation).
|
||||
|
||||
* 4.0.2 (September 21, 2015)
|
||||
|
||||
This bugfix release addresses a few bugs specific to heap profiling.
|
||||
|
||||
Bug fixes:
|
||||
- Fix ixallocx_prof_sample() to never modify nor create sampled small
|
||||
allocations. xallocx() is in general incapable of moving small allocations,
|
||||
so this fix removes buggy code without loss of generality.
|
||||
- Fix irallocx_prof_sample() to always allocate large regions, even when
|
||||
alignment is non-zero.
|
||||
- Fix prof_alloc_rollback() to read tdata from thread-specific data rather
|
||||
than dereferencing a potentially invalid tctx.
|
||||
|
||||
* 4.0.1 (September 15, 2015)
|
||||
|
||||
This is a bugfix release that is somewhat high risk due to the amount of
|
||||
refactoring required to address deep xallocx() problems. As a side effect of
|
||||
these fixes, xallocx() now tries harder to partially fulfill requests for
|
||||
optional extra space. Note that a couple of minor heap profiling
|
||||
optimizations are included, but these are better thought of as performance
|
||||
fixes that were integral to disovering most of the other bugs.
|
||||
|
||||
Optimizations:
|
||||
- Avoid a chunk metadata read in arena_prof_tctx_set(), since it is in the
|
||||
fast path when heap profiling is enabled. Additionally, split a special
|
||||
case out into arena_prof_tctx_reset(), which also avoids chunk metadata
|
||||
reads.
|
||||
- Optimize irallocx_prof() to optimistically update the sampler state. The
|
||||
prior implementation appears to have been a holdover from when
|
||||
rallocx()/xallocx() functionality was combined as rallocm().
|
||||
|
||||
Bug fixes:
|
||||
- Fix TLS configuration such that it is enabled by default for platforms on
|
||||
which it works correctly.
|
||||
- Fix arenas_cache_cleanup() and arena_get_hard() to handle
|
||||
allocation/deallocation within the application's thread-specific data
|
||||
cleanup functions even after arenas_cache is torn down.
|
||||
- Fix xallocx() bugs related to size+extra exceeding HUGE_MAXCLASS.
|
||||
- Fix chunk purge hook calls for in-place huge shrinking reallocation to
|
||||
specify the old chunk size rather than the new chunk size. This bug caused
|
||||
no correctness issues for the default chunk purge function, but was
|
||||
visible to custom functions set via the "arena.<i>.chunk_hooks" mallctl.
|
||||
- Fix heap profiling bugs:
|
||||
+ Fix heap profiling to distinguish among otherwise identical sample sites
|
||||
with interposed resets (triggered via the "prof.reset" mallctl). This bug
|
||||
could cause data structure corruption that would most likely result in a
|
||||
segfault.
|
||||
+ Fix irealloc_prof() to prof_alloc_rollback() on OOM.
|
||||
+ Make one call to prof_active_get_unlocked() per allocation event, and use
|
||||
the result throughout the relevant functions that handle an allocation
|
||||
event. Also add a missing check in prof_realloc(). These fixes protect
|
||||
allocation events against concurrent prof_active changes.
|
||||
+ Fix ixallocx_prof() to pass usize_max and zero to ixallocx_prof_sample()
|
||||
in the correct order.
|
||||
+ Fix prof_realloc() to call prof_free_sampled_object() after calling
|
||||
prof_malloc_sample_object(). Prior to this fix, if tctx and old_tctx were
|
||||
the same, the tctx could have been prematurely destroyed.
|
||||
- Fix portability bugs:
|
||||
+ Don't bitshift by negative amounts when encoding/decoding run sizes in
|
||||
chunk header maps. This affected systems with page sizes greater than 8
|
||||
KiB.
|
||||
+ Rename index_t to szind_t to avoid an existing type on Solaris.
|
||||
+ Add JEMALLOC_CXX_THROW to the memalign() function prototype, in order to
|
||||
match glibc and avoid compilation errors when including both
|
||||
jemalloc/jemalloc.h and malloc.h in C++ code.
|
||||
+ Don't assume that /bin/sh is appropriate when running size_classes.sh
|
||||
during configuration.
|
||||
+ Consider __sparcv9 a synonym for __sparc64__ when defining LG_QUANTUM.
|
||||
+ Link tests to librt if it contains clock_gettime(2).
|
||||
|
||||
* 4.0.0 (August 17, 2015)
|
||||
|
||||
This version contains many speed and space optimizations, both minor and
|
||||
major. The major themes are generalization, unification, and simplification.
|
||||
Although many of these optimizations cause no visible behavior change, their
|
||||
cumulative effect is substantial.
|
||||
|
||||
New features:
|
||||
- Normalize size class spacing to be consistent across the complete size
|
||||
range. By default there are four size classes per size doubling, but this
|
||||
is now configurable via the --with-lg-size-class-group option. Also add the
|
||||
--with-lg-page, --with-lg-page-sizes, --with-lg-quantum, and
|
||||
--with-lg-tiny-min options, which can be used to tweak page and size class
|
||||
settings. Impacts:
|
||||
+ Worst case performance for incrementally growing/shrinking reallocation
|
||||
is improved because there are far fewer size classes, and therefore
|
||||
copying happens less often.
|
||||
+ Internal fragmentation is limited to 20% for all but the smallest size
|
||||
classes (those less than four times the quantum). (1B + 4 KiB)
|
||||
and (1B + 4 MiB) previously suffered nearly 50% internal fragmentation.
|
||||
+ Chunk fragmentation tends to be lower because there are fewer distinct run
|
||||
sizes to pack.
|
||||
- Add support for explicit tcaches. The "tcache.create", "tcache.flush", and
|
||||
"tcache.destroy" mallctls control tcache lifetime and flushing, and the
|
||||
MALLOCX_TCACHE(tc) and MALLOCX_TCACHE_NONE flags to the *allocx() API
|
||||
control which tcache is used for each operation.
|
||||
- Implement per thread heap profiling, as well as the ability to
|
||||
enable/disable heap profiling on a per thread basis. Add the "prof.reset",
|
||||
"prof.lg_sample", "thread.prof.name", "thread.prof.active",
|
||||
"opt.prof_thread_active_init", "prof.thread_active_init", and
|
||||
"thread.prof.active" mallctls.
|
||||
- Add support for per arena application-specified chunk allocators, configured
|
||||
via the "arena.<i>.chunk_hooks" mallctl.
|
||||
- Refactor huge allocation to be managed by arenas, so that arenas now
|
||||
function as general purpose independent allocators. This is important in
|
||||
the context of user-specified chunk allocators, aside from the scalability
|
||||
benefits. Related new statistics:
|
||||
+ The "stats.arenas.<i>.huge.allocated", "stats.arenas.<i>.huge.nmalloc",
|
||||
"stats.arenas.<i>.huge.ndalloc", and "stats.arenas.<i>.huge.nrequests"
|
||||
mallctls provide high level per arena huge allocation statistics.
|
||||
+ The "arenas.nhchunks", "arenas.hchunk.<i>.size",
|
||||
"stats.arenas.<i>.hchunks.<j>.nmalloc",
|
||||
"stats.arenas.<i>.hchunks.<j>.ndalloc",
|
||||
"stats.arenas.<i>.hchunks.<j>.nrequests", and
|
||||
"stats.arenas.<i>.hchunks.<j>.curhchunks" mallctls provide per size class
|
||||
statistics.
|
||||
- Add the 'util' column to malloc_stats_print() output, which reports the
|
||||
proportion of available regions that are currently in use for each small
|
||||
size class.
|
||||
- Add "alloc" and "free" modes for for junk filling (see the "opt.junk"
|
||||
mallctl), so that it is possible to separately enable junk filling for
|
||||
allocation versus deallocation.
|
||||
- Add the jemalloc-config script, which provides information about how
|
||||
jemalloc was configured, and how to integrate it into application builds.
|
||||
- Add metadata statistics, which are accessible via the "stats.metadata",
|
||||
"stats.arenas.<i>.metadata.mapped", and
|
||||
"stats.arenas.<i>.metadata.allocated" mallctls.
|
||||
- Add the "stats.resident" mallctl, which reports the upper limit of
|
||||
physically resident memory mapped by the allocator.
|
||||
- Add per arena control over unused dirty page purging, via the
|
||||
"arenas.lg_dirty_mult", "arena.<i>.lg_dirty_mult", and
|
||||
"stats.arenas.<i>.lg_dirty_mult" mallctls.
|
||||
- Add the "prof.gdump" mallctl, which makes it possible to toggle the gdump
|
||||
feature on/off during program execution.
|
||||
- Add sdallocx(), which implements sized deallocation. The primary
|
||||
optimization over dallocx() is the removal of a metadata read, which often
|
||||
suffers an L1 cache miss.
|
||||
- Add missing header includes in jemalloc/jemalloc.h, so that applications
|
||||
only have to #include <jemalloc/jemalloc.h>.
|
||||
- Add support for additional platforms:
|
||||
+ Bitrig
|
||||
+ Cygwin
|
||||
+ DragonFlyBSD
|
||||
+ iOS
|
||||
+ OpenBSD
|
||||
+ OpenRISC/or1k
|
||||
|
||||
Optimizations:
|
||||
- Maintain dirty runs in per arena LRUs rather than in per arena trees of
|
||||
dirty-run-containing chunks. In practice this change significantly reduces
|
||||
dirty page purging volume.
|
||||
- Integrate whole chunks into the unused dirty page purging machinery. This
|
||||
reduces the cost of repeated huge allocation/deallocation, because it
|
||||
effectively introduces a cache of chunks.
|
||||
- Split the arena chunk map into two separate arrays, in order to increase
|
||||
cache locality for the frequently accessed bits.
|
||||
- Move small run metadata out of runs, into arena chunk headers. This reduces
|
||||
run fragmentation, smaller runs reduce external fragmentation for small size
|
||||
classes, and packed (less uniformly aligned) metadata layout improves CPU
|
||||
cache set distribution.
|
||||
- Randomly distribute large allocation base pointer alignment relative to page
|
||||
boundaries in order to more uniformly utilize CPU cache sets. This can be
|
||||
disabled via the --disable-cache-oblivious configure option, and queried via
|
||||
the "config.cache_oblivious" mallctl.
|
||||
- Micro-optimize the fast paths for the public API functions.
|
||||
- Refactor thread-specific data to reside in a single structure. This assures
|
||||
that only a single TLS read is necessary per call into the public API.
|
||||
- Implement in-place huge allocation growing and shrinking.
|
||||
- Refactor rtree (radix tree for chunk lookups) to be lock-free, and make
|
||||
additional optimizations that reduce maximum lookup depth to one or two
|
||||
levels. This resolves what was a concurrency bottleneck for per arena huge
|
||||
allocation, because a global data structure is critical for determining
|
||||
which arenas own which huge allocations.
|
||||
|
||||
Incompatible changes:
|
||||
- Replace --enable-cc-silence with --disable-cc-silence to suppress spurious
|
||||
warnings by default.
|
||||
- Assure that the constness of malloc_usable_size()'s return type matches that
|
||||
of the system implementation.
|
||||
- Change the heap profile dump format to support per thread heap profiling,
|
||||
rename pprof to jeprof, and enhance it with the --thread=<n> option. As a
|
||||
result, the bundled jeprof must now be used rather than the upstream
|
||||
(gperftools) pprof.
|
||||
- Disable "opt.prof_final" by default, in order to avoid atexit(3), which can
|
||||
internally deadlock on some platforms.
|
||||
- Change the "arenas.nlruns" mallctl type from size_t to unsigned.
|
||||
- Replace the "stats.arenas.<i>.bins.<j>.allocated" mallctl with
|
||||
"stats.arenas.<i>.bins.<j>.curregs".
|
||||
- Ignore MALLOC_CONF in set{uid,gid,cap} binaries.
|
||||
- Ignore MALLOCX_ARENA(a) in dallocx(), in favor of using the
|
||||
MALLOCX_TCACHE(tc) and MALLOCX_TCACHE_NONE flags to control tcache usage.
|
||||
|
||||
Removed features:
|
||||
- Remove the *allocm() API, which is superseded by the *allocx() API.
|
||||
- Remove the --enable-dss options, and make dss non-optional on all platforms
|
||||
which support sbrk(2).
|
||||
- Remove the "arenas.purge" mallctl, which was obsoleted by the
|
||||
"arena.<i>.purge" mallctl in 3.1.0.
|
||||
- Remove the unnecessary "opt.valgrind" mallctl; jemalloc automatically
|
||||
detects whether it is running inside Valgrind.
|
||||
- Remove the "stats.huge.allocated", "stats.huge.nmalloc", and
|
||||
"stats.huge.ndalloc" mallctls.
|
||||
- Remove the --enable-mremap option.
|
||||
- Remove the "stats.chunks.current", "stats.chunks.total", and
|
||||
"stats.chunks.high" mallctls.
|
||||
|
||||
Bug fixes:
|
||||
- Fix the cactive statistic to decrease (rather than increase) when active
|
||||
memory decreases. This regression was first released in 3.5.0.
|
||||
- Fix OOM handling in memalign() and valloc(). A variant of this bug existed
|
||||
in all releases since 2.0.0, which introduced these functions.
|
||||
- Fix an OOM-related regression in arena_tcache_fill_small(), which could
|
||||
cause cache corruption on OOM. This regression was present in all releases
|
||||
from 2.2.0 through 3.6.0.
|
||||
- Fix size class overflow handling for malloc(), posix_memalign(), memalign(),
|
||||
calloc(), and realloc() when profiling is enabled.
|
||||
- Fix the "arena.<i>.dss" mallctl to return an error if "primary" or
|
||||
"secondary" precedence is specified, but sbrk(2) is not supported.
|
||||
- Fix fallback lg_floor() implementations to handle extremely large inputs.
|
||||
- Ensure the default purgeable zone is after the default zone on OS X.
|
||||
- Fix latent bugs in atomic_*().
|
||||
- Fix the "arena.<i>.dss" mallctl to handle read-only calls.
|
||||
- Fix tls_model configuration to enable the initial-exec model when possible.
|
||||
- Mark malloc_conf as a weak symbol so that the application can override it.
|
||||
- Correctly detect glibc's adaptive pthread mutexes.
|
||||
- Fix the --without-export configure option.
|
||||
|
||||
* 3.6.0 (March 31, 2014)
|
||||
|
||||
This version contains a critical bug fix for a regression present in 3.5.0 and
|
||||
@@ -481,7 +21,7 @@ brevity. Much more detail can be found in the git revision history:
|
||||
backtracing to be reliable.
|
||||
- Use dss allocation precedence for huge allocations as well as small/large
|
||||
allocations.
|
||||
- Fix test assertion failure message formatting. This bug did not manifest on
|
||||
- Fix test assertion failure message formatting. This bug did not manifect on
|
||||
x86_64 systems because of implementation subtleties in va_list.
|
||||
- Fix inconsequential test failures for hash and SFMT code.
|
||||
|
||||
@@ -976,7 +516,7 @@ brevity. Much more detail can be found in the git revision history:
|
||||
- Make it possible for the application to manually flush a thread's cache, via
|
||||
the "tcache.flush" mallctl.
|
||||
- Base maximum dirty page count on proportion of active memory.
|
||||
- Compute various additional run-time statistics, including per size class
|
||||
- Compute various addtional run-time statistics, including per size class
|
||||
statistics for large objects.
|
||||
- Expose malloc_stats_print(), which can be called repeatedly by the
|
||||
application.
|
||||
|
||||
174
INSTALL
174
INSTALL
@@ -1,23 +1,10 @@
|
||||
Building and installing a packaged release of jemalloc can be as simple as
|
||||
typing the following while in the root directory of the source tree:
|
||||
Building and installing jemalloc can be as simple as typing the following while
|
||||
in the root directory of the source tree:
|
||||
|
||||
./configure
|
||||
make
|
||||
make install
|
||||
|
||||
If building from unpackaged developer sources, the simplest command sequence
|
||||
that might work is:
|
||||
|
||||
./autogen.sh
|
||||
make dist
|
||||
make
|
||||
make install
|
||||
|
||||
Note that documentation is not built by the default target because doing so
|
||||
would create a dependency on xsltproc in packaged releases, hence the
|
||||
requirement to either run 'make dist' or avoid installing docs via the various
|
||||
install_* targets documented below.
|
||||
|
||||
=== Advanced configuration =====================================================
|
||||
|
||||
The 'configure' script supports numerous options that allow control of which
|
||||
@@ -35,10 +22,6 @@ any of the following arguments (not a definitive list) to 'configure':
|
||||
will cause files to be installed into /usr/local/include, /usr/local/lib,
|
||||
and /usr/local/man.
|
||||
|
||||
--with-version=<major>.<minor>.<bugfix>-<nrev>-g<gid>
|
||||
Use the specified version string rather than trying to generate one (if in
|
||||
a git repository) or use existing the VERSION file (if present).
|
||||
|
||||
--with-rpath=<colon-separated-rpath>
|
||||
Embed one or more library paths, so that libjemalloc can find the libraries
|
||||
it is linked to. This works only on ELF-based systems.
|
||||
@@ -73,7 +56,7 @@ any of the following arguments (not a definitive list) to 'configure':
|
||||
replace the "malloc", "calloc", etc. symbols.
|
||||
|
||||
--without-export
|
||||
Don't export public APIs. This can be useful when building jemalloc as a
|
||||
Don't export public APIs. This can be useful when building jemalloc as a
|
||||
static library, or to avoid exporting public APIs when using the zone
|
||||
allocator on OSX.
|
||||
|
||||
@@ -88,18 +71,10 @@ any of the following arguments (not a definitive list) to 'configure':
|
||||
versions of jemalloc can coexist in the same installation directory. For
|
||||
example, libjemalloc.so.0 becomes libjemalloc<suffix>.so.0.
|
||||
|
||||
--with-malloc-conf=<malloc_conf>
|
||||
Embed <malloc_conf> as a run-time options string that is processed prior to
|
||||
the malloc_conf global variable, the /etc/malloc.conf symlink, and the
|
||||
MALLOC_CONF environment variable. For example, to change the default chunk
|
||||
size to 256 KiB:
|
||||
|
||||
--with-malloc-conf=lg_chunk:18
|
||||
|
||||
--disable-cc-silence
|
||||
Disable code that silences non-useful compiler warnings. This is mainly
|
||||
useful during development when auditing the set of warnings that are being
|
||||
silenced.
|
||||
--enable-cc-silence
|
||||
Enable code that silences non-useful compiler warnings. This is helpful
|
||||
when trying to tell serious warnings from those due to compiler
|
||||
limitations, but it potentially incurs a performance penalty.
|
||||
|
||||
--enable-debug
|
||||
Enable assertions and validation code. This incurs a substantial
|
||||
@@ -119,15 +94,15 @@ any of the following arguments (not a definitive list) to 'configure':
|
||||
there are interactions between the various coverage targets, so it is
|
||||
usually advisable to run 'make clean' between repeated code coverage runs.
|
||||
|
||||
--enable-ivsalloc
|
||||
Enable validation code, which verifies that pointers reside within
|
||||
jemalloc-owned chunks before dereferencing them. This incurs a substantial
|
||||
performance hit.
|
||||
|
||||
--disable-stats
|
||||
Disable statistics gathering functionality. See the "opt.stats_print"
|
||||
option documentation for usage details.
|
||||
|
||||
--enable-ivsalloc
|
||||
Enable validation code, which verifies that pointers reside within
|
||||
jemalloc-owned chunks before dereferencing them. This incurs a minor
|
||||
performance hit.
|
||||
|
||||
--enable-prof
|
||||
Enable heap profiling and leak detection functionality. See the "opt.prof"
|
||||
option documentation for usage details. When enabled, there are several
|
||||
@@ -157,6 +132,12 @@ any of the following arguments (not a definitive list) to 'configure':
|
||||
released in bulk, thus reducing the total number of mutex operations. See
|
||||
the "opt.tcache" option for usage details.
|
||||
|
||||
--enable-mremap
|
||||
Enable huge realloc() via mremap(2). mremap() is disabled by default
|
||||
because the flavor used is specific to Linux, which has a quirk in its
|
||||
virtual memory allocation algorithm that causes semi-permanent VM map holes
|
||||
under normal jemalloc operation.
|
||||
|
||||
--disable-munmap
|
||||
Disable virtual memory deallocation via munmap(2); instead keep track of
|
||||
the virtual memory for later use. munmap() is disabled by default (i.e.
|
||||
@@ -164,6 +145,10 @@ any of the following arguments (not a definitive list) to 'configure':
|
||||
memory allocation algorithm that causes semi-permanent VM map holes under
|
||||
normal jemalloc operation.
|
||||
|
||||
--enable-dss
|
||||
Enable support for page allocation/deallocation via sbrk(2), in addition to
|
||||
mmap(2).
|
||||
|
||||
--disable-fill
|
||||
Disable support for junk/zero filling of memory, quarantine, and redzones.
|
||||
See the "opt.junk", "opt.zero", "opt.quarantine", and "opt.redzone" option
|
||||
@@ -172,8 +157,11 @@ any of the following arguments (not a definitive list) to 'configure':
|
||||
--disable-valgrind
|
||||
Disable support for Valgrind.
|
||||
|
||||
--disable-experimental
|
||||
Disable support for the experimental API (*allocm()).
|
||||
|
||||
--disable-zone-allocator
|
||||
Disable zone allocator for Darwin. This means jemalloc won't be hooked as
|
||||
Disable zone allocator for Darwin. This means jemalloc won't be hooked as
|
||||
the default allocator on OSX/iOS.
|
||||
|
||||
--enable-utrace
|
||||
@@ -197,111 +185,10 @@ any of the following arguments (not a definitive list) to 'configure':
|
||||
thread-local variables via the __thread keyword. If TLS is available,
|
||||
jemalloc uses it for several purposes.
|
||||
|
||||
--disable-cache-oblivious
|
||||
Disable cache-oblivious large allocation alignment for large allocation
|
||||
requests with no alignment constraints. If this feature is disabled, all
|
||||
large allocations are page-aligned as an implementation artifact, which can
|
||||
severely harm CPU cache utilization. However, the cache-oblivious layout
|
||||
comes at the cost of one extra page per large allocation, which in the
|
||||
most extreme case increases physical memory usage for the 16 KiB size class
|
||||
to 20 KiB.
|
||||
|
||||
--disable-syscall
|
||||
Disable use of syscall(2) rather than {open,read,write,close}(2). This is
|
||||
intended as a workaround for systems that place security limitations on
|
||||
syscall(2).
|
||||
|
||||
--with-xslroot=<path>
|
||||
Specify where to find DocBook XSL stylesheets when building the
|
||||
documentation.
|
||||
|
||||
--with-lg-page=<lg-page>
|
||||
Specify the base 2 log of the system page size. This option is only useful
|
||||
when cross compiling, since the configure script automatically determines
|
||||
the host's page size by default.
|
||||
|
||||
--with-lg-page-sizes=<lg-page-sizes>
|
||||
Specify the comma-separated base 2 logs of the page sizes to support. This
|
||||
option may be useful when cross-compiling in combination with
|
||||
--with-lg-page, but its primary use case is for integration with FreeBSD's
|
||||
libc, wherein jemalloc is embedded.
|
||||
|
||||
--with-lg-size-class-group=<lg-size-class-group>
|
||||
Specify the base 2 log of how many size classes to use for each doubling in
|
||||
size. By default jemalloc uses <lg-size-class-group>=2, which results in
|
||||
e.g. the following size classes:
|
||||
|
||||
[...], 64,
|
||||
80, 96, 112, 128,
|
||||
160, [...]
|
||||
|
||||
<lg-size-class-group>=3 results in e.g. the following size classes:
|
||||
|
||||
[...], 64,
|
||||
72, 80, 88, 96, 104, 112, 120, 128,
|
||||
144, [...]
|
||||
|
||||
The minimal <lg-size-class-group>=0 causes jemalloc to only provide size
|
||||
classes that are powers of 2:
|
||||
|
||||
[...],
|
||||
64,
|
||||
128,
|
||||
256,
|
||||
[...]
|
||||
|
||||
An implementation detail currently limits the total number of small size
|
||||
classes to 255, and a compilation error will result if the
|
||||
<lg-size-class-group> you specify cannot be supported. The limit is
|
||||
roughly <lg-size-class-group>=4, depending on page size.
|
||||
|
||||
--with-lg-quantum=<lg-quantum>
|
||||
Specify the base 2 log of the minimum allocation alignment. jemalloc needs
|
||||
to know the minimum alignment that meets the following C standard
|
||||
requirement (quoted from the April 12, 2011 draft of the C11 standard):
|
||||
|
||||
The pointer returned if the allocation succeeds is suitably aligned so
|
||||
that it may be assigned to a pointer to any type of object with a
|
||||
fundamental alignment requirement and then used to access such an object
|
||||
or an array of such objects in the space allocated [...]
|
||||
|
||||
This setting is architecture-specific, and although jemalloc includes known
|
||||
safe values for the most commonly used modern architectures, there is a
|
||||
wrinkle related to GNU libc (glibc) that may impact your choice of
|
||||
<lg-quantum>. On most modern architectures, this mandates 16-byte alignment
|
||||
(<lg-quantum>=4), but the glibc developers chose not to meet this
|
||||
requirement for performance reasons. An old discussion can be found at
|
||||
https://sourceware.org/bugzilla/show_bug.cgi?id=206 . Unlike glibc,
|
||||
jemalloc does follow the C standard by default (caveat: jemalloc
|
||||
technically cheats if --with-lg-tiny-min is smaller than
|
||||
--with-lg-quantum), but the fact that Linux systems already work around
|
||||
this allocator noncompliance means that it is generally safe in practice to
|
||||
let jemalloc's minimum alignment follow glibc's lead. If you specify
|
||||
--with-lg-quantum=3 during configuration, jemalloc will provide additional
|
||||
size classes that are not 16-byte-aligned (24, 40, and 56, assuming
|
||||
--with-lg-size-class-group=2).
|
||||
|
||||
--with-lg-tiny-min=<lg-tiny-min>
|
||||
Specify the base 2 log of the minimum tiny size class to support. Tiny
|
||||
size classes are powers of 2 less than the quantum, and are only
|
||||
incorporated if <lg-tiny-min> is less than <lg-quantum> (see
|
||||
--with-lg-quantum). Tiny size classes technically violate the C standard
|
||||
requirement for minimum alignment, and crashes could conceivably result if
|
||||
the compiler were to generate instructions that made alignment assumptions,
|
||||
both because illegal instruction traps could result, and because accesses
|
||||
could straddle page boundaries and cause segmentation faults due to
|
||||
accessing unmapped addresses.
|
||||
|
||||
The default of <lg-tiny-min>=3 works well in practice even on architectures
|
||||
that technically require 16-byte alignment, probably for the same reason
|
||||
--with-lg-quantum=3 works. Smaller tiny size classes can, and will, cause
|
||||
crashes (see https://bugzilla.mozilla.org/show_bug.cgi?id=691003 for an
|
||||
example).
|
||||
|
||||
This option is rarely useful, and is mainly provided as documentation of a
|
||||
subtle implementation detail. If you do use this option, specify a
|
||||
value in [3, ..., <lg-quantum>].
|
||||
|
||||
The following environment variables (not a definitive list) impact configure's
|
||||
behavior:
|
||||
|
||||
@@ -332,15 +219,6 @@ LDFLAGS="?"
|
||||
PATH="?"
|
||||
'configure' uses this to find programs.
|
||||
|
||||
In some cases it may be necessary to work around configuration results that do
|
||||
not match reality. For example, Linux 4.5 added support for the MADV_FREE flag
|
||||
to madvise(2), which can cause problems if building on a host with MADV_FREE
|
||||
support and deploying to a target without. To work around this, use a cache
|
||||
file to override the relevant configuration variable defined in configure.ac,
|
||||
e.g.:
|
||||
|
||||
echo "je_cv_madv_free=no" > config.cache && ./configure -C
|
||||
|
||||
=== Advanced compilation =======================================================
|
||||
|
||||
To build only parts of jemalloc, use the following targets:
|
||||
|
||||
212
Makefile.in
212
Makefile.in
@@ -24,8 +24,7 @@ abs_objroot := @abs_objroot@
|
||||
|
||||
# Build parameters.
|
||||
CPPFLAGS := @CPPFLAGS@ -I$(srcroot)include -I$(objroot)include
|
||||
EXTRA_CFLAGS := @EXTRA_CFLAGS@
|
||||
CFLAGS := @CFLAGS@ $(EXTRA_CFLAGS)
|
||||
CFLAGS := @CFLAGS@
|
||||
LDFLAGS := @LDFLAGS@
|
||||
EXTRA_LDFLAGS := @EXTRA_LDFLAGS@
|
||||
LIBS := @LIBS@
|
||||
@@ -43,29 +42,23 @@ XSLTPROC := @XSLTPROC@
|
||||
AUTOCONF := @AUTOCONF@
|
||||
_RPATH = @RPATH@
|
||||
RPATH = $(if $(1),$(call _RPATH,$(1)))
|
||||
cfghdrs_in := $(addprefix $(srcroot),@cfghdrs_in@)
|
||||
cfghdrs_in := @cfghdrs_in@
|
||||
cfghdrs_out := @cfghdrs_out@
|
||||
cfgoutputs_in := $(addprefix $(srcroot),@cfgoutputs_in@)
|
||||
cfgoutputs_in := @cfgoutputs_in@
|
||||
cfgoutputs_out := @cfgoutputs_out@
|
||||
enable_autogen := @enable_autogen@
|
||||
enable_code_coverage := @enable_code_coverage@
|
||||
enable_prof := @enable_prof@
|
||||
enable_valgrind := @enable_valgrind@
|
||||
enable_experimental := @enable_experimental@
|
||||
enable_zone_allocator := @enable_zone_allocator@
|
||||
MALLOC_CONF := @JEMALLOC_CPREFIX@MALLOC_CONF
|
||||
link_whole_archive := @link_whole_archive@
|
||||
DSO_LDFLAGS = @DSO_LDFLAGS@
|
||||
SOREV = @SOREV@
|
||||
PIC_CFLAGS = @PIC_CFLAGS@
|
||||
CTARGET = @CTARGET@
|
||||
LDTARGET = @LDTARGET@
|
||||
TEST_LD_MODE = @TEST_LD_MODE@
|
||||
MKLIB = @MKLIB@
|
||||
AR = @AR@
|
||||
ARFLAGS = @ARFLAGS@
|
||||
CC_MM = @CC_MM@
|
||||
LM := @LM@
|
||||
INSTALL = @INSTALL@
|
||||
|
||||
ifeq (macho, $(ABI))
|
||||
TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH="$(objroot)lib"
|
||||
@@ -80,39 +73,16 @@ endif
|
||||
LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix)
|
||||
|
||||
# Lists of files.
|
||||
BINS := $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh $(objroot)bin/jeprof
|
||||
BINS := $(srcroot)bin/pprof $(objroot)bin/jemalloc.sh
|
||||
C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h
|
||||
C_SRCS := $(srcroot)src/jemalloc.c \
|
||||
$(srcroot)src/arena.c \
|
||||
$(srcroot)src/atomic.c \
|
||||
$(srcroot)src/base.c \
|
||||
$(srcroot)src/bitmap.c \
|
||||
$(srcroot)src/chunk.c \
|
||||
$(srcroot)src/chunk_dss.c \
|
||||
$(srcroot)src/chunk_mmap.c \
|
||||
$(srcroot)src/ckh.c \
|
||||
$(srcroot)src/ctl.c \
|
||||
$(srcroot)src/extent.c \
|
||||
$(srcroot)src/hash.c \
|
||||
$(srcroot)src/huge.c \
|
||||
$(srcroot)src/mb.c \
|
||||
$(srcroot)src/mutex.c \
|
||||
$(srcroot)src/nstime.c \
|
||||
$(srcroot)src/pages.c \
|
||||
$(srcroot)src/prng.c \
|
||||
$(srcroot)src/prof.c \
|
||||
$(srcroot)src/quarantine.c \
|
||||
$(srcroot)src/rtree.c \
|
||||
$(srcroot)src/stats.c \
|
||||
$(srcroot)src/spin.c \
|
||||
$(srcroot)src/tcache.c \
|
||||
$(srcroot)src/ticker.c \
|
||||
$(srcroot)src/tsd.c \
|
||||
$(srcroot)src/util.c \
|
||||
$(srcroot)src/witness.c
|
||||
ifeq ($(enable_valgrind), 1)
|
||||
C_SRCS += $(srcroot)src/valgrind.c
|
||||
endif
|
||||
C_SRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c \
|
||||
$(srcroot)src/atomic.c $(srcroot)src/base.c $(srcroot)src/bitmap.c \
|
||||
$(srcroot)src/chunk.c $(srcroot)src/chunk_dss.c \
|
||||
$(srcroot)src/chunk_mmap.c $(srcroot)src/ckh.c $(srcroot)src/ctl.c \
|
||||
$(srcroot)src/extent.c $(srcroot)src/hash.c $(srcroot)src/huge.c \
|
||||
$(srcroot)src/mb.c $(srcroot)src/mutex.c $(srcroot)src/prof.c \
|
||||
$(srcroot)src/quarantine.c $(srcroot)src/rtree.c $(srcroot)src/stats.c \
|
||||
$(srcroot)src/tcache.c $(srcroot)src/util.c $(srcroot)src/tsd.c
|
||||
ifeq ($(enable_zone_allocator), 1)
|
||||
C_SRCS += $(srcroot)src/zone.c
|
||||
endif
|
||||
@@ -128,83 +98,53 @@ DSOS := $(objroot)lib/$(LIBJEMALLOC).$(SOREV)
|
||||
ifneq ($(SOREV),$(SO))
|
||||
DSOS += $(objroot)lib/$(LIBJEMALLOC).$(SO)
|
||||
endif
|
||||
ifeq (1, $(link_whole_archive))
|
||||
LJEMALLOC := -Wl,--whole-archive -L$(objroot)lib -l$(LIBJEMALLOC) -Wl,--no-whole-archive
|
||||
else
|
||||
LJEMALLOC := $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
|
||||
endif
|
||||
PC := $(objroot)jemalloc.pc
|
||||
MAN3 := $(objroot)doc/jemalloc$(install_suffix).3
|
||||
DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml
|
||||
DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(objroot)%.html)
|
||||
DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(objroot)%.3)
|
||||
DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.html)
|
||||
DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.3)
|
||||
DOCS := $(DOCS_HTML) $(DOCS_MAN3)
|
||||
C_TESTLIB_SRCS := $(srcroot)test/src/btalloc.c $(srcroot)test/src/btalloc_0.c \
|
||||
$(srcroot)test/src/btalloc_1.c $(srcroot)test/src/math.c \
|
||||
$(srcroot)test/src/mtx.c $(srcroot)test/src/mq.c \
|
||||
C_TESTLIB_SRCS := $(srcroot)test/src/math.c $(srcroot)test/src/mtx.c \
|
||||
$(srcroot)test/src/SFMT.c $(srcroot)test/src/test.c \
|
||||
$(srcroot)test/src/thd.c $(srcroot)test/src/timer.c
|
||||
ifeq (1, $(link_whole_archive))
|
||||
C_UTIL_INTEGRATION_SRCS :=
|
||||
else
|
||||
C_UTIL_INTEGRATION_SRCS := $(srcroot)src/nstime.c $(srcroot)src/util.c
|
||||
endif
|
||||
TESTS_UNIT := \
|
||||
$(srcroot)test/unit/a0.c \
|
||||
$(srcroot)test/unit/arena_reset.c \
|
||||
$(srcroot)test/unit/atomic.c \
|
||||
$(srcroot)test/unit/bitmap.c \
|
||||
$(srcroot)test/src/thd.c
|
||||
C_UTIL_INTEGRATION_SRCS := $(srcroot)src/util.c
|
||||
TESTS_UNIT := $(srcroot)test/unit/bitmap.c \
|
||||
$(srcroot)test/unit/ckh.c \
|
||||
$(srcroot)test/unit/decay.c \
|
||||
$(srcroot)test/unit/fork.c \
|
||||
$(srcroot)test/unit/hash.c \
|
||||
$(srcroot)test/unit/junk.c \
|
||||
$(srcroot)test/unit/junk_alloc.c \
|
||||
$(srcroot)test/unit/junk_free.c \
|
||||
$(srcroot)test/unit/lg_chunk.c \
|
||||
$(srcroot)test/unit/mallctl.c \
|
||||
$(srcroot)test/unit/math.c \
|
||||
$(srcroot)test/unit/mq.c \
|
||||
$(srcroot)test/unit/mtx.c \
|
||||
$(srcroot)test/unit/pack.c \
|
||||
$(srcroot)test/unit/pages.c \
|
||||
$(srcroot)test/unit/ph.c \
|
||||
$(srcroot)test/unit/prng.c \
|
||||
$(srcroot)test/unit/prof_accum.c \
|
||||
$(srcroot)test/unit/prof_active.c \
|
||||
$(srcroot)test/unit/prof_gdump.c \
|
||||
$(srcroot)test/unit/prof_idump.c \
|
||||
$(srcroot)test/unit/prof_reset.c \
|
||||
$(srcroot)test/unit/prof_thread_name.c \
|
||||
$(srcroot)test/unit/ql.c \
|
||||
$(srcroot)test/unit/qr.c \
|
||||
$(srcroot)test/unit/quarantine.c \
|
||||
$(srcroot)test/unit/rb.c \
|
||||
$(srcroot)test/unit/rtree.c \
|
||||
$(srcroot)test/unit/run_quantize.c \
|
||||
$(srcroot)test/unit/SFMT.c \
|
||||
$(srcroot)test/unit/size_classes.c \
|
||||
$(srcroot)test/unit/smoothstep.c \
|
||||
$(srcroot)test/unit/stats.c \
|
||||
$(srcroot)test/unit/ticker.c \
|
||||
$(srcroot)test/unit/nstime.c \
|
||||
$(srcroot)test/unit/tsd.c \
|
||||
$(srcroot)test/unit/util.c \
|
||||
$(srcroot)test/unit/witness.c \
|
||||
$(srcroot)test/unit/zero.c
|
||||
TESTS_UNIT_AUX := $(srcroot)test/unit/prof_accum_a.c \
|
||||
$(srcroot)test/unit/prof_accum_b.c
|
||||
TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
|
||||
$(srcroot)test/integration/allocated.c \
|
||||
$(srcroot)test/integration/sdallocx.c \
|
||||
$(srcroot)test/integration/mallocx.c \
|
||||
$(srcroot)test/integration/MALLOCX_ARENA.c \
|
||||
$(srcroot)test/integration/overflow.c \
|
||||
$(srcroot)test/integration/mremap.c \
|
||||
$(srcroot)test/integration/posix_memalign.c \
|
||||
$(srcroot)test/integration/rallocx.c \
|
||||
$(srcroot)test/integration/thread_arena.c \
|
||||
$(srcroot)test/integration/thread_tcache_enabled.c \
|
||||
$(srcroot)test/integration/xallocx.c \
|
||||
$(srcroot)test/integration/chunk.c
|
||||
TESTS_STRESS := $(srcroot)test/stress/microbench.c
|
||||
$(srcroot)test/integration/xallocx.c
|
||||
ifeq ($(enable_experimental), 1)
|
||||
TESTS_INTEGRATION += $(srcroot)test/integration/allocm.c \
|
||||
$(srcroot)test/integration/MALLOCX_ARENA.c \
|
||||
$(srcroot)test/integration/rallocm.c
|
||||
endif
|
||||
TESTS_STRESS :=
|
||||
TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_STRESS)
|
||||
|
||||
C_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.$(O))
|
||||
@@ -217,9 +157,10 @@ C_TESTLIB_STRESS_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.stress.$(O))
|
||||
C_TESTLIB_OBJS := $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(C_TESTLIB_STRESS_OBJS)
|
||||
|
||||
TESTS_UNIT_OBJS := $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%.$(O))
|
||||
TESTS_UNIT_AUX_OBJS := $(TESTS_UNIT_AUX:$(srcroot)%.c=$(objroot)%.$(O))
|
||||
TESTS_INTEGRATION_OBJS := $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%.$(O))
|
||||
TESTS_STRESS_OBJS := $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%.$(O))
|
||||
TESTS_OBJS := $(TESTS_UNIT_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_STRESS_OBJS)
|
||||
TESTS_OBJS := $(TESTS_UNIT_OBJS) $(TESTS_UNIT_AUX_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_STRESS_OBJS)
|
||||
|
||||
.PHONY: all dist build_doc_html build_doc_man build_doc
|
||||
.PHONY: install_bin install_include install_lib
|
||||
@@ -233,10 +174,10 @@ all: build_lib
|
||||
|
||||
dist: build_doc
|
||||
|
||||
$(objroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl
|
||||
$(srcroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl
|
||||
$(XSLTPROC) -o $@ $(objroot)doc/html.xsl $<
|
||||
|
||||
$(objroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl
|
||||
$(srcroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl
|
||||
$(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $<
|
||||
|
||||
build_doc_html: $(DOCS_HTML)
|
||||
@@ -268,12 +209,18 @@ $(C_TESTLIB_STRESS_OBJS): $(objroot)test/src/%.stress.$(O): $(srcroot)test/src/%
|
||||
$(C_TESTLIB_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST -DJEMALLOC_STRESS_TESTLIB
|
||||
$(C_TESTLIB_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include
|
||||
$(TESTS_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST
|
||||
$(TESTS_UNIT_AUX_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST
|
||||
define make-unit-link-dep
|
||||
$(1): TESTS_UNIT_LINK_OBJS += $(2)
|
||||
$(1): $(2)
|
||||
endef
|
||||
$(foreach test, $(TESTS_UNIT:$(srcroot)test/unit/%.c=$(objroot)test/unit/%$(EXE)), $(eval $(call make-unit-link-dep,$(test),$(filter $(test:%=%_a.$(O)) $(test:%=%_b.$(O)),$(TESTS_UNIT_AUX_OBJS)))))
|
||||
$(TESTS_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST
|
||||
$(TESTS_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST
|
||||
$(TESTS_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c
|
||||
$(TESTS_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include
|
||||
ifneq ($(IMPORTLIB),$(SO))
|
||||
$(C_OBJS) $(C_JET_OBJS): CPPFLAGS += -DDLLEXPORT
|
||||
$(C_OBJS): CPPFLAGS += -DDLLEXPORT
|
||||
endif
|
||||
|
||||
ifndef CC_MM
|
||||
@@ -282,7 +229,7 @@ HEADER_DIRS = $(srcroot)include/jemalloc/internal \
|
||||
$(objroot)include/jemalloc $(objroot)include/jemalloc/internal
|
||||
HEADERS = $(wildcard $(foreach dir,$(HEADER_DIRS),$(dir)/*.h))
|
||||
$(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): $(HEADERS)
|
||||
$(TESTS_OBJS): $(objroot)test/include/test/jemalloc_test.h
|
||||
$(TESTS_OBJS): $(objroot)test/unit/jemalloc_test.h
|
||||
endif
|
||||
|
||||
$(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): %.$(O):
|
||||
@@ -312,69 +259,62 @@ $(STATIC_LIBS):
|
||||
|
||||
$(objroot)test/unit/%$(EXE): $(objroot)test/unit/%.$(O) $(TESTS_UNIT_LINK_OBJS) $(C_JET_OBJS) $(C_TESTLIB_UNIT_OBJS)
|
||||
@mkdir -p $(@D)
|
||||
$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LDFLAGS) $(filter-out -lm,$(LIBS)) $(LM) $(EXTRA_LDFLAGS)
|
||||
$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS)
|
||||
|
||||
$(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
|
||||
@mkdir -p $(@D)
|
||||
$(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -lpthread,$(LIBS))) $(LM) $(EXTRA_LDFLAGS)
|
||||
$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(filter -lpthread,$(LIBS))) -lm $(EXTRA_LDFLAGS)
|
||||
|
||||
$(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_STRESS_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
|
||||
@mkdir -p $(@D)
|
||||
$(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) $(LM) $(EXTRA_LDFLAGS)
|
||||
$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS)
|
||||
|
||||
build_lib_shared: $(DSOS)
|
||||
build_lib_static: $(STATIC_LIBS)
|
||||
build_lib: build_lib_shared build_lib_static
|
||||
|
||||
install_bin:
|
||||
$(INSTALL) -d $(BINDIR)
|
||||
install -d $(BINDIR)
|
||||
@for b in $(BINS); do \
|
||||
echo "$(INSTALL) -m 755 $$b $(BINDIR)"; \
|
||||
$(INSTALL) -m 755 $$b $(BINDIR); \
|
||||
echo "install -m 755 $$b $(BINDIR)"; \
|
||||
install -m 755 $$b $(BINDIR); \
|
||||
done
|
||||
|
||||
install_include:
|
||||
$(INSTALL) -d $(INCLUDEDIR)/jemalloc
|
||||
install -d $(INCLUDEDIR)/jemalloc
|
||||
@for h in $(C_HDRS); do \
|
||||
echo "$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc"; \
|
||||
$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc; \
|
||||
echo "install -m 644 $$h $(INCLUDEDIR)/jemalloc"; \
|
||||
install -m 644 $$h $(INCLUDEDIR)/jemalloc; \
|
||||
done
|
||||
|
||||
install_lib_shared: $(DSOS)
|
||||
$(INSTALL) -d $(LIBDIR)
|
||||
$(INSTALL) -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR)
|
||||
install -d $(LIBDIR)
|
||||
install -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR)
|
||||
ifneq ($(SOREV),$(SO))
|
||||
ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO)
|
||||
endif
|
||||
|
||||
install_lib_static: $(STATIC_LIBS)
|
||||
$(INSTALL) -d $(LIBDIR)
|
||||
install -d $(LIBDIR)
|
||||
@for l in $(STATIC_LIBS); do \
|
||||
echo "$(INSTALL) -m 755 $$l $(LIBDIR)"; \
|
||||
$(INSTALL) -m 755 $$l $(LIBDIR); \
|
||||
echo "install -m 755 $$l $(LIBDIR)"; \
|
||||
install -m 755 $$l $(LIBDIR); \
|
||||
done
|
||||
|
||||
install_lib_pc: $(PC)
|
||||
$(INSTALL) -d $(LIBDIR)/pkgconfig
|
||||
@for l in $(PC); do \
|
||||
echo "$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig"; \
|
||||
$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig; \
|
||||
done
|
||||
|
||||
install_lib: install_lib_shared install_lib_static install_lib_pc
|
||||
install_lib: install_lib_shared install_lib_static
|
||||
|
||||
install_doc_html:
|
||||
$(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix)
|
||||
install -d $(DATADIR)/doc/jemalloc$(install_suffix)
|
||||
@for d in $(DOCS_HTML); do \
|
||||
echo "$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \
|
||||
$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \
|
||||
echo "install -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \
|
||||
install -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \
|
||||
done
|
||||
|
||||
install_doc_man:
|
||||
$(INSTALL) -d $(MANDIR)/man3
|
||||
install -d $(MANDIR)/man3
|
||||
@for d in $(DOCS_MAN3); do \
|
||||
echo "$(INSTALL) -m 644 $$d $(MANDIR)/man3"; \
|
||||
$(INSTALL) -m 644 $$d $(MANDIR)/man3; \
|
||||
echo "install -m 644 $$d $(MANDIR)/man3"; \
|
||||
install -m 644 $$d $(MANDIR)/man3; \
|
||||
done
|
||||
|
||||
install_doc: install_doc_html install_doc_man
|
||||
@@ -390,27 +330,18 @@ check_unit_dir:
|
||||
@mkdir -p $(objroot)test/unit
|
||||
check_integration_dir:
|
||||
@mkdir -p $(objroot)test/integration
|
||||
stress_dir:
|
||||
check_stress_dir:
|
||||
@mkdir -p $(objroot)test/stress
|
||||
check_dir: check_unit_dir check_integration_dir
|
||||
check_dir: check_unit_dir check_integration_dir check_stress_dir
|
||||
|
||||
check_unit: tests_unit check_unit_dir
|
||||
$(MALLOC_CONF)="purge:ratio" $(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%)
|
||||
$(MALLOC_CONF)="purge:decay" $(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%)
|
||||
check_integration_prof: tests_integration check_integration_dir
|
||||
ifeq ($(enable_prof), 1)
|
||||
$(MALLOC_CONF)="prof:true" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
$(MALLOC_CONF)="prof:true,prof_active:false" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
endif
|
||||
check_integration_decay: tests_integration check_integration_dir
|
||||
$(MALLOC_CONF)="purge:decay,decay_time:-1" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
$(MALLOC_CONF)="purge:decay,decay_time:0" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
$(MALLOC_CONF)="purge:decay" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
$(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%)
|
||||
check_integration: tests_integration check_integration_dir
|
||||
$(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
stress: tests_stress stress_dir
|
||||
check_stress: tests_stress check_stress_dir
|
||||
$(SHELL) $(objroot)test/test.sh $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%)
|
||||
check: check_unit check_integration check_integration_decay check_integration_prof
|
||||
check: tests check_dir
|
||||
$(SHELL) $(objroot)test/test.sh $(TESTS:$(srcroot)%.c=$(objroot)%)
|
||||
|
||||
ifeq ($(enable_code_coverage), 1)
|
||||
coverage_unit: check_unit
|
||||
@@ -424,7 +355,7 @@ coverage_integration: check_integration
|
||||
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS)
|
||||
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS)
|
||||
|
||||
coverage_stress: stress
|
||||
coverage_stress: check_stress
|
||||
$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
|
||||
$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
|
||||
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS)
|
||||
@@ -469,9 +400,8 @@ clean:
|
||||
rm -f $(objroot)*.gcov.*
|
||||
|
||||
distclean: clean
|
||||
rm -f $(objroot)bin/jemalloc-config
|
||||
rm -rf $(objroot)autom4te.cache
|
||||
rm -f $(objroot)bin/jemalloc.sh
|
||||
rm -f $(objroot)bin/jeprof
|
||||
rm -f $(objroot)config.log
|
||||
rm -f $(objroot)config.status
|
||||
rm -f $(objroot)config.stamp
|
||||
@@ -480,7 +410,7 @@ distclean: clean
|
||||
|
||||
relclean: distclean
|
||||
rm -f $(objroot)configure
|
||||
rm -f $(objroot)VERSION
|
||||
rm -f $(srcroot)VERSION
|
||||
rm -f $(DOCS_HTML)
|
||||
rm -f $(DOCS_MAN3)
|
||||
|
||||
|
||||
2
README
2
README
@@ -17,4 +17,4 @@ jemalloc.
|
||||
|
||||
The ChangeLog file contains a brief summary of changes for each release.
|
||||
|
||||
URL: http://jemalloc.net/
|
||||
URL: http://www.canonware.com/jemalloc/
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage:
|
||||
@BINDIR@/jemalloc-config <option>
|
||||
Options:
|
||||
--help | -h : Print usage.
|
||||
--version : Print jemalloc version.
|
||||
--revision : Print shared library revision number.
|
||||
--config : Print configure options used to build jemalloc.
|
||||
--prefix : Print installation directory prefix.
|
||||
--bindir : Print binary installation directory.
|
||||
--datadir : Print data installation directory.
|
||||
--includedir : Print include installation directory.
|
||||
--libdir : Print library installation directory.
|
||||
--mandir : Print manual page installation directory.
|
||||
--cc : Print compiler used to build jemalloc.
|
||||
--cflags : Print compiler flags used to build jemalloc.
|
||||
--cppflags : Print preprocessor flags used to build jemalloc.
|
||||
--ldflags : Print library flags used to build jemalloc.
|
||||
--libs : Print libraries jemalloc was linked against.
|
||||
EOF
|
||||
}
|
||||
|
||||
prefix="@prefix@"
|
||||
exec_prefix="@exec_prefix@"
|
||||
|
||||
case "$1" in
|
||||
--help | -h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
--version)
|
||||
echo "@jemalloc_version@"
|
||||
;;
|
||||
--revision)
|
||||
echo "@rev@"
|
||||
;;
|
||||
--config)
|
||||
echo "@CONFIG@"
|
||||
;;
|
||||
--prefix)
|
||||
echo "@PREFIX@"
|
||||
;;
|
||||
--bindir)
|
||||
echo "@BINDIR@"
|
||||
;;
|
||||
--datadir)
|
||||
echo "@DATADIR@"
|
||||
;;
|
||||
--includedir)
|
||||
echo "@INCLUDEDIR@"
|
||||
;;
|
||||
--libdir)
|
||||
echo "@LIBDIR@"
|
||||
;;
|
||||
--mandir)
|
||||
echo "@MANDIR@"
|
||||
;;
|
||||
--cc)
|
||||
echo "@CC@"
|
||||
;;
|
||||
--cflags)
|
||||
echo "@CFLAGS@"
|
||||
;;
|
||||
--cppflags)
|
||||
echo "@CPPFLAGS@"
|
||||
;;
|
||||
--ldflags)
|
||||
echo "@LDFLAGS@ @EXTRA_LDFLAGS@"
|
||||
;;
|
||||
--libs)
|
||||
echo "@LIBS@"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
esac
|
||||
661
bin/jeprof.in → bin/pprof
Normal file → Executable file
661
bin/jeprof.in → bin/pprof
Normal file → Executable file
File diff suppressed because it is too large
Load Diff
354
build-aux/config.guess → config.guess
vendored
354
build-aux/config.guess → config.guess
vendored
@@ -1,8 +1,8 @@
|
||||
#! /bin/sh
|
||||
# Attempt to guess a canonical system name.
|
||||
# Copyright 1992-2016 Free Software Foundation, Inc.
|
||||
# Copyright 1992-2013 Free Software Foundation, Inc.
|
||||
|
||||
timestamp='2016-10-02'
|
||||
timestamp='2013-06-10'
|
||||
|
||||
# This file is free software; you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by
|
||||
@@ -24,12 +24,12 @@ timestamp='2016-10-02'
|
||||
# program. This Exception is an additional permission under section 7
|
||||
# of the GNU General Public License, version 3 ("GPLv3").
|
||||
#
|
||||
# Originally written by Per Bothner; maintained since 2000 by Ben Elliston.
|
||||
# Originally written by Per Bothner.
|
||||
#
|
||||
# You can get the latest version of this script from:
|
||||
# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
|
||||
# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
|
||||
#
|
||||
# Please send patches to <config-patches@gnu.org>.
|
||||
# Please send patches with a ChangeLog entry to config-patches@gnu.org.
|
||||
|
||||
|
||||
me=`echo "$0" | sed -e 's,.*/,,'`
|
||||
@@ -50,7 +50,7 @@ version="\
|
||||
GNU config.guess ($timestamp)
|
||||
|
||||
Originally written by Per Bothner.
|
||||
Copyright 1992-2016 Free Software Foundation, Inc.
|
||||
Copyright 1992-2013 Free Software Foundation, Inc.
|
||||
|
||||
This is free software; see the source for copying conditions. There is NO
|
||||
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
|
||||
@@ -149,7 +149,7 @@ Linux|GNU|GNU/*)
|
||||
LIBC=gnu
|
||||
#endif
|
||||
EOF
|
||||
eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`
|
||||
eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'`
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -168,29 +168,19 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
|
||||
# Note: NetBSD doesn't particularly care about the vendor
|
||||
# portion of the name. We always set it to "unknown".
|
||||
sysctl="sysctl -n hw.machine_arch"
|
||||
UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \
|
||||
/sbin/$sysctl 2>/dev/null || \
|
||||
/usr/sbin/$sysctl 2>/dev/null || \
|
||||
echo unknown)`
|
||||
UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \
|
||||
/usr/sbin/$sysctl 2>/dev/null || echo unknown)`
|
||||
case "${UNAME_MACHINE_ARCH}" in
|
||||
armeb) machine=armeb-unknown ;;
|
||||
arm*) machine=arm-unknown ;;
|
||||
sh3el) machine=shl-unknown ;;
|
||||
sh3eb) machine=sh-unknown ;;
|
||||
sh5el) machine=sh5le-unknown ;;
|
||||
earmv*)
|
||||
arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'`
|
||||
endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'`
|
||||
machine=${arch}${endian}-unknown
|
||||
;;
|
||||
*) machine=${UNAME_MACHINE_ARCH}-unknown ;;
|
||||
esac
|
||||
# The Operating System including object format, if it has switched
|
||||
# to ELF recently (or will in the future) and ABI.
|
||||
# to ELF recently, or will in the future.
|
||||
case "${UNAME_MACHINE_ARCH}" in
|
||||
earm*)
|
||||
os=netbsdelf
|
||||
;;
|
||||
arm*|i386|m68k|ns32k|sh3*|sparc|vax)
|
||||
eval $set_cc_for_build
|
||||
if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \
|
||||
@@ -207,13 +197,6 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
|
||||
os=netbsd
|
||||
;;
|
||||
esac
|
||||
# Determine ABI tags.
|
||||
case "${UNAME_MACHINE_ARCH}" in
|
||||
earm*)
|
||||
expr='s/^earmv[0-9]/-eabi/;s/eb$//'
|
||||
abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"`
|
||||
;;
|
||||
esac
|
||||
# The OS release
|
||||
# Debian GNU/NetBSD machines have a different userland, and
|
||||
# thus, need a distinct triplet. However, they do not need
|
||||
@@ -224,13 +207,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
|
||||
release='-gnu'
|
||||
;;
|
||||
*)
|
||||
release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d. -f1,2`
|
||||
release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
|
||||
;;
|
||||
esac
|
||||
# Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM:
|
||||
# contains redundant information, the shorter form:
|
||||
# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used.
|
||||
echo "${machine}-${os}${release}${abi}"
|
||||
echo "${machine}-${os}${release}"
|
||||
exit ;;
|
||||
*:Bitrig:*:*)
|
||||
UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'`
|
||||
@@ -240,10 +223,6 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
|
||||
UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'`
|
||||
echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE}
|
||||
exit ;;
|
||||
*:LibertyBSD:*:*)
|
||||
UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'`
|
||||
echo ${UNAME_MACHINE_ARCH}-unknown-libertybsd${UNAME_RELEASE}
|
||||
exit ;;
|
||||
*:ekkoBSD:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE}
|
||||
exit ;;
|
||||
@@ -256,9 +235,6 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
|
||||
*:MirBSD:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE}
|
||||
exit ;;
|
||||
*:Sortix:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-sortix
|
||||
exit ;;
|
||||
alpha:OSF1:*:*)
|
||||
case $UNAME_RELEASE in
|
||||
*4.0)
|
||||
@@ -275,42 +251,42 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
|
||||
ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1`
|
||||
case "$ALPHA_CPU_TYPE" in
|
||||
"EV4 (21064)")
|
||||
UNAME_MACHINE=alpha ;;
|
||||
UNAME_MACHINE="alpha" ;;
|
||||
"EV4.5 (21064)")
|
||||
UNAME_MACHINE=alpha ;;
|
||||
UNAME_MACHINE="alpha" ;;
|
||||
"LCA4 (21066/21068)")
|
||||
UNAME_MACHINE=alpha ;;
|
||||
UNAME_MACHINE="alpha" ;;
|
||||
"EV5 (21164)")
|
||||
UNAME_MACHINE=alphaev5 ;;
|
||||
UNAME_MACHINE="alphaev5" ;;
|
||||
"EV5.6 (21164A)")
|
||||
UNAME_MACHINE=alphaev56 ;;
|
||||
UNAME_MACHINE="alphaev56" ;;
|
||||
"EV5.6 (21164PC)")
|
||||
UNAME_MACHINE=alphapca56 ;;
|
||||
UNAME_MACHINE="alphapca56" ;;
|
||||
"EV5.7 (21164PC)")
|
||||
UNAME_MACHINE=alphapca57 ;;
|
||||
UNAME_MACHINE="alphapca57" ;;
|
||||
"EV6 (21264)")
|
||||
UNAME_MACHINE=alphaev6 ;;
|
||||
UNAME_MACHINE="alphaev6" ;;
|
||||
"EV6.7 (21264A)")
|
||||
UNAME_MACHINE=alphaev67 ;;
|
||||
UNAME_MACHINE="alphaev67" ;;
|
||||
"EV6.8CB (21264C)")
|
||||
UNAME_MACHINE=alphaev68 ;;
|
||||
UNAME_MACHINE="alphaev68" ;;
|
||||
"EV6.8AL (21264B)")
|
||||
UNAME_MACHINE=alphaev68 ;;
|
||||
UNAME_MACHINE="alphaev68" ;;
|
||||
"EV6.8CX (21264D)")
|
||||
UNAME_MACHINE=alphaev68 ;;
|
||||
UNAME_MACHINE="alphaev68" ;;
|
||||
"EV6.9A (21264/EV69A)")
|
||||
UNAME_MACHINE=alphaev69 ;;
|
||||
UNAME_MACHINE="alphaev69" ;;
|
||||
"EV7 (21364)")
|
||||
UNAME_MACHINE=alphaev7 ;;
|
||||
UNAME_MACHINE="alphaev7" ;;
|
||||
"EV7.9 (21364A)")
|
||||
UNAME_MACHINE=alphaev79 ;;
|
||||
UNAME_MACHINE="alphaev79" ;;
|
||||
esac
|
||||
# A Pn.n version is a patched version.
|
||||
# A Vn.n version is a released version.
|
||||
# A Tn.n version is a released field test version.
|
||||
# A Xn.n version is an unreleased experimental baselevel.
|
||||
# 1.2 uses "1.2" for uname -r.
|
||||
echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
|
||||
echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
|
||||
# Reset EXIT trap before exiting to avoid spurious non-zero exit code.
|
||||
exitcode=$?
|
||||
trap '' 0
|
||||
@@ -383,16 +359,16 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
|
||||
exit ;;
|
||||
i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*)
|
||||
eval $set_cc_for_build
|
||||
SUN_ARCH=i386
|
||||
SUN_ARCH="i386"
|
||||
# If there is a compiler, see if it is configured for 64-bit objects.
|
||||
# Note that the Sun cc does not turn __LP64__ into 1 like gcc does.
|
||||
# This test works for both compilers.
|
||||
if [ "$CC_FOR_BUILD" != no_compiler_found ]; then
|
||||
if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
|
||||
if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \
|
||||
(CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
|
||||
(CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
|
||||
grep IS_64BIT_ARCH >/dev/null
|
||||
then
|
||||
SUN_ARCH=x86_64
|
||||
SUN_ARCH="x86_64"
|
||||
fi
|
||||
fi
|
||||
echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
|
||||
@@ -417,7 +393,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
|
||||
exit ;;
|
||||
sun*:*:4.2BSD:*)
|
||||
UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null`
|
||||
test "x${UNAME_RELEASE}" = x && UNAME_RELEASE=3
|
||||
test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3
|
||||
case "`/bin/arch`" in
|
||||
sun3)
|
||||
echo m68k-sun-sunos${UNAME_RELEASE}
|
||||
@@ -603,9 +579,8 @@ EOF
|
||||
else
|
||||
IBM_ARCH=powerpc
|
||||
fi
|
||||
if [ -x /usr/bin/lslpp ] ; then
|
||||
IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc |
|
||||
awk -F: '{ print $3 }' | sed s/[0-9]*$/0/`
|
||||
if [ -x /usr/bin/oslevel ] ; then
|
||||
IBM_REV=`/usr/bin/oslevel`
|
||||
else
|
||||
IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE}
|
||||
fi
|
||||
@@ -642,13 +617,13 @@ EOF
|
||||
sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null`
|
||||
sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
|
||||
case "${sc_cpu_version}" in
|
||||
523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0
|
||||
528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1
|
||||
523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0
|
||||
528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1
|
||||
532) # CPU_PA_RISC2_0
|
||||
case "${sc_kernel_bits}" in
|
||||
32) HP_ARCH=hppa2.0n ;;
|
||||
64) HP_ARCH=hppa2.0w ;;
|
||||
'') HP_ARCH=hppa2.0 ;; # HP-UX 10.20
|
||||
32) HP_ARCH="hppa2.0n" ;;
|
||||
64) HP_ARCH="hppa2.0w" ;;
|
||||
'') HP_ARCH="hppa2.0" ;; # HP-UX 10.20
|
||||
esac ;;
|
||||
esac
|
||||
fi
|
||||
@@ -687,11 +662,11 @@ EOF
|
||||
exit (0);
|
||||
}
|
||||
EOF
|
||||
(CCOPTS="" $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy`
|
||||
(CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy`
|
||||
test -z "$HP_ARCH" && HP_ARCH=hppa
|
||||
fi ;;
|
||||
esac
|
||||
if [ ${HP_ARCH} = hppa2.0w ]
|
||||
if [ ${HP_ARCH} = "hppa2.0w" ]
|
||||
then
|
||||
eval $set_cc_for_build
|
||||
|
||||
@@ -704,12 +679,12 @@ EOF
|
||||
# $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess
|
||||
# => hppa64-hp-hpux11.23
|
||||
|
||||
if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) |
|
||||
if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) |
|
||||
grep -q __LP64__
|
||||
then
|
||||
HP_ARCH=hppa2.0w
|
||||
HP_ARCH="hppa2.0w"
|
||||
else
|
||||
HP_ARCH=hppa64
|
||||
HP_ARCH="hppa64"
|
||||
fi
|
||||
fi
|
||||
echo ${HP_ARCH}-hp-hpux${HPUX_REV}
|
||||
@@ -814,14 +789,14 @@ EOF
|
||||
echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
|
||||
exit ;;
|
||||
F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*)
|
||||
FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
|
||||
FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
|
||||
FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
|
||||
FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
|
||||
FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'`
|
||||
echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
|
||||
exit ;;
|
||||
5000:UNIX_System_V:4.*:*)
|
||||
FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
|
||||
FUJITSU_REL=`echo ${UNAME_RELEASE} | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'`
|
||||
FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
|
||||
FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'`
|
||||
echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
|
||||
exit ;;
|
||||
i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*)
|
||||
@@ -851,7 +826,7 @@ EOF
|
||||
*:MINGW*:*)
|
||||
echo ${UNAME_MACHINE}-pc-mingw32
|
||||
exit ;;
|
||||
*:MSYS*:*)
|
||||
i*:MSYS*:*)
|
||||
echo ${UNAME_MACHINE}-pc-msys
|
||||
exit ;;
|
||||
i*:windows32*:*)
|
||||
@@ -903,7 +878,7 @@ EOF
|
||||
exit ;;
|
||||
*:GNU/*:*:*)
|
||||
# other systems with GNU libc and userland
|
||||
echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC}
|
||||
echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC}
|
||||
exit ;;
|
||||
i*86:Minix:*:*)
|
||||
echo ${UNAME_MACHINE}-pc-minix
|
||||
@@ -926,7 +901,7 @@ EOF
|
||||
EV68*) UNAME_MACHINE=alphaev68 ;;
|
||||
esac
|
||||
objdump --private-headers /bin/sh | grep -q ld.so.1
|
||||
if test "$?" = 0 ; then LIBC=gnulibc1 ; fi
|
||||
if test "$?" = 0 ; then LIBC="gnulibc1" ; fi
|
||||
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
|
||||
exit ;;
|
||||
arc:Linux:*:* | arceb:Linux:*:*)
|
||||
@@ -957,9 +932,6 @@ EOF
|
||||
crisv32:Linux:*:*)
|
||||
echo ${UNAME_MACHINE}-axis-linux-${LIBC}
|
||||
exit ;;
|
||||
e2k:Linux:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
|
||||
exit ;;
|
||||
frv:Linux:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
|
||||
exit ;;
|
||||
@@ -972,9 +944,6 @@ EOF
|
||||
ia64:Linux:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
|
||||
exit ;;
|
||||
k1om:Linux:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
|
||||
exit ;;
|
||||
m32r*:Linux:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
|
||||
exit ;;
|
||||
@@ -1000,13 +969,10 @@ EOF
|
||||
eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'`
|
||||
test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; }
|
||||
;;
|
||||
mips64el:Linux:*:*)
|
||||
or1k:Linux:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
|
||||
exit ;;
|
||||
openrisc*:Linux:*:*)
|
||||
echo or1k-unknown-linux-${LIBC}
|
||||
exit ;;
|
||||
or32:Linux:*:* | or1k*:Linux:*:*)
|
||||
or32:Linux:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
|
||||
exit ;;
|
||||
padre:Linux:*:*)
|
||||
@@ -1035,9 +1001,6 @@ EOF
|
||||
ppcle:Linux:*:*)
|
||||
echo powerpcle-unknown-linux-${LIBC}
|
||||
exit ;;
|
||||
riscv32:Linux:*:* | riscv64:Linux:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
|
||||
exit ;;
|
||||
s390:Linux:*:* | s390x:Linux:*:*)
|
||||
echo ${UNAME_MACHINE}-ibm-linux-${LIBC}
|
||||
exit ;;
|
||||
@@ -1057,7 +1020,7 @@ EOF
|
||||
echo ${UNAME_MACHINE}-dec-linux-${LIBC}
|
||||
exit ;;
|
||||
x86_64:Linux:*:*)
|
||||
echo ${UNAME_MACHINE}-pc-linux-${LIBC}
|
||||
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
|
||||
exit ;;
|
||||
xtensa*:Linux:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
|
||||
@@ -1136,7 +1099,7 @@ EOF
|
||||
# uname -m prints for DJGPP always 'pc', but it prints nothing about
|
||||
# the processor, so we play safe by assuming i586.
|
||||
# Note: whatever this is, it MUST be the same as what config.sub
|
||||
# prints for the "djgpp" host, or else GDB configure will decide that
|
||||
# prints for the "djgpp" host, or else GDB configury will decide that
|
||||
# this is a cross-build.
|
||||
echo i586-pc-msdosdjgpp
|
||||
exit ;;
|
||||
@@ -1285,9 +1248,6 @@ EOF
|
||||
SX-8R:SUPER-UX:*:*)
|
||||
echo sx8r-nec-superux${UNAME_RELEASE}
|
||||
exit ;;
|
||||
SX-ACE:SUPER-UX:*:*)
|
||||
echo sxace-nec-superux${UNAME_RELEASE}
|
||||
exit ;;
|
||||
Power*:Rhapsody:*:*)
|
||||
echo powerpc-apple-rhapsody${UNAME_RELEASE}
|
||||
exit ;;
|
||||
@@ -1300,32 +1260,22 @@ EOF
|
||||
if test "$UNAME_PROCESSOR" = unknown ; then
|
||||
UNAME_PROCESSOR=powerpc
|
||||
fi
|
||||
if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then
|
||||
if [ "$CC_FOR_BUILD" != no_compiler_found ]; then
|
||||
if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
|
||||
(CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
|
||||
grep IS_64BIT_ARCH >/dev/null
|
||||
then
|
||||
case $UNAME_PROCESSOR in
|
||||
i386) UNAME_PROCESSOR=x86_64 ;;
|
||||
powerpc) UNAME_PROCESSOR=powerpc64 ;;
|
||||
esac
|
||||
fi
|
||||
if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
|
||||
if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
|
||||
(CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
|
||||
grep IS_64BIT_ARCH >/dev/null
|
||||
then
|
||||
case $UNAME_PROCESSOR in
|
||||
i386) UNAME_PROCESSOR=x86_64 ;;
|
||||
powerpc) UNAME_PROCESSOR=powerpc64 ;;
|
||||
esac
|
||||
fi
|
||||
elif test "$UNAME_PROCESSOR" = i386 ; then
|
||||
# Avoid executing cc on OS X 10.9, as it ships with a stub
|
||||
# that puts up a graphical alert prompting to install
|
||||
# developer tools. Any system running Mac OS X 10.7 or
|
||||
# later (Darwin 11 and later) is required to have a 64-bit
|
||||
# processor. This is not true of the ARM version of Darwin
|
||||
# that Apple uses in portable devices.
|
||||
UNAME_PROCESSOR=x86_64
|
||||
fi
|
||||
echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE}
|
||||
exit ;;
|
||||
*:procnto*:*:* | *:QNX:[0123456789]*:*)
|
||||
UNAME_PROCESSOR=`uname -p`
|
||||
if test "$UNAME_PROCESSOR" = x86; then
|
||||
if test "$UNAME_PROCESSOR" = "x86"; then
|
||||
UNAME_PROCESSOR=i386
|
||||
UNAME_MACHINE=pc
|
||||
fi
|
||||
@@ -1356,7 +1306,7 @@ EOF
|
||||
# "uname -m" is not consistent, so use $cputype instead. 386
|
||||
# is converted to i386 for consistency with other x86
|
||||
# operating systems.
|
||||
if test "$cputype" = 386; then
|
||||
if test "$cputype" = "386"; then
|
||||
UNAME_MACHINE=i386
|
||||
else
|
||||
UNAME_MACHINE="$cputype"
|
||||
@@ -1398,7 +1348,7 @@ EOF
|
||||
echo i386-pc-xenix
|
||||
exit ;;
|
||||
i*86:skyos:*:*)
|
||||
echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE} | sed -e 's/ .*$//'`
|
||||
echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//'
|
||||
exit ;;
|
||||
i*86:rdos:*:*)
|
||||
echo ${UNAME_MACHINE}-pc-rdos
|
||||
@@ -1409,25 +1359,171 @@ EOF
|
||||
x86_64:VMkernel:*:*)
|
||||
echo ${UNAME_MACHINE}-unknown-esx
|
||||
exit ;;
|
||||
amd64:Isilon\ OneFS:*:*)
|
||||
echo x86_64-unknown-onefs
|
||||
exit ;;
|
||||
esac
|
||||
|
||||
eval $set_cc_for_build
|
||||
cat >$dummy.c <<EOF
|
||||
#ifdef _SEQUENT_
|
||||
# include <sys/types.h>
|
||||
# include <sys/utsname.h>
|
||||
#endif
|
||||
main ()
|
||||
{
|
||||
#if defined (sony)
|
||||
#if defined (MIPSEB)
|
||||
/* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed,
|
||||
I don't know.... */
|
||||
printf ("mips-sony-bsd\n"); exit (0);
|
||||
#else
|
||||
#include <sys/param.h>
|
||||
printf ("m68k-sony-newsos%s\n",
|
||||
#ifdef NEWSOS4
|
||||
"4"
|
||||
#else
|
||||
""
|
||||
#endif
|
||||
); exit (0);
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined (__arm) && defined (__acorn) && defined (__unix)
|
||||
printf ("arm-acorn-riscix\n"); exit (0);
|
||||
#endif
|
||||
|
||||
#if defined (hp300) && !defined (hpux)
|
||||
printf ("m68k-hp-bsd\n"); exit (0);
|
||||
#endif
|
||||
|
||||
#if defined (NeXT)
|
||||
#if !defined (__ARCHITECTURE__)
|
||||
#define __ARCHITECTURE__ "m68k"
|
||||
#endif
|
||||
int version;
|
||||
version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
|
||||
if (version < 4)
|
||||
printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version);
|
||||
else
|
||||
printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version);
|
||||
exit (0);
|
||||
#endif
|
||||
|
||||
#if defined (MULTIMAX) || defined (n16)
|
||||
#if defined (UMAXV)
|
||||
printf ("ns32k-encore-sysv\n"); exit (0);
|
||||
#else
|
||||
#if defined (CMU)
|
||||
printf ("ns32k-encore-mach\n"); exit (0);
|
||||
#else
|
||||
printf ("ns32k-encore-bsd\n"); exit (0);
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined (__386BSD__)
|
||||
printf ("i386-pc-bsd\n"); exit (0);
|
||||
#endif
|
||||
|
||||
#if defined (sequent)
|
||||
#if defined (i386)
|
||||
printf ("i386-sequent-dynix\n"); exit (0);
|
||||
#endif
|
||||
#if defined (ns32000)
|
||||
printf ("ns32k-sequent-dynix\n"); exit (0);
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined (_SEQUENT_)
|
||||
struct utsname un;
|
||||
|
||||
uname(&un);
|
||||
|
||||
if (strncmp(un.version, "V2", 2) == 0) {
|
||||
printf ("i386-sequent-ptx2\n"); exit (0);
|
||||
}
|
||||
if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */
|
||||
printf ("i386-sequent-ptx1\n"); exit (0);
|
||||
}
|
||||
printf ("i386-sequent-ptx\n"); exit (0);
|
||||
|
||||
#endif
|
||||
|
||||
#if defined (vax)
|
||||
# if !defined (ultrix)
|
||||
# include <sys/param.h>
|
||||
# if defined (BSD)
|
||||
# if BSD == 43
|
||||
printf ("vax-dec-bsd4.3\n"); exit (0);
|
||||
# else
|
||||
# if BSD == 199006
|
||||
printf ("vax-dec-bsd4.3reno\n"); exit (0);
|
||||
# else
|
||||
printf ("vax-dec-bsd\n"); exit (0);
|
||||
# endif
|
||||
# endif
|
||||
# else
|
||||
printf ("vax-dec-bsd\n"); exit (0);
|
||||
# endif
|
||||
# else
|
||||
printf ("vax-dec-ultrix\n"); exit (0);
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if defined (alliant) && defined (i860)
|
||||
printf ("i860-alliant-bsd\n"); exit (0);
|
||||
#endif
|
||||
|
||||
exit (1);
|
||||
}
|
||||
EOF
|
||||
|
||||
$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` &&
|
||||
{ echo "$SYSTEM_NAME"; exit; }
|
||||
|
||||
# Apollos put the system type in the environment.
|
||||
|
||||
test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; }
|
||||
|
||||
# Convex versions that predate uname can use getsysinfo(1)
|
||||
|
||||
if [ -x /usr/convex/getsysinfo ]
|
||||
then
|
||||
case `getsysinfo -f cpu_type` in
|
||||
c1*)
|
||||
echo c1-convex-bsd
|
||||
exit ;;
|
||||
c2*)
|
||||
if getsysinfo -f scalar_acc
|
||||
then echo c32-convex-bsd
|
||||
else echo c2-convex-bsd
|
||||
fi
|
||||
exit ;;
|
||||
c34*)
|
||||
echo c34-convex-bsd
|
||||
exit ;;
|
||||
c38*)
|
||||
echo c38-convex-bsd
|
||||
exit ;;
|
||||
c4*)
|
||||
echo c4-convex-bsd
|
||||
exit ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
cat >&2 <<EOF
|
||||
$0: unable to guess system type
|
||||
|
||||
This script (version $timestamp), has failed to recognize the
|
||||
operating system you are using. If your script is old, overwrite
|
||||
config.guess and config.sub with the latest versions from:
|
||||
This script, last modified $timestamp, has failed to recognize
|
||||
the operating system you are using. It is advised that you
|
||||
download the most up to date version of the config scripts from
|
||||
|
||||
http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
|
||||
http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
|
||||
and
|
||||
http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
|
||||
http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
|
||||
|
||||
If $0 has already been updated, send the following data and any
|
||||
information you think might be pertinent to config-patches@gnu.org to
|
||||
provide the necessary information to handle your system.
|
||||
If the version you run ($0) is already up to date, please
|
||||
send the following data and any information you think might be
|
||||
pertinent to <config-patches@gnu.org> in order to provide the needed
|
||||
information to handle your system.
|
||||
|
||||
config.guess timestamp = $timestamp
|
||||
|
||||
84
build-aux/config.sub → config.sub
vendored
84
build-aux/config.sub → config.sub
vendored
@@ -1,8 +1,8 @@
|
||||
#! /bin/sh
|
||||
# Configuration validation subroutine script.
|
||||
# Copyright 1992-2016 Free Software Foundation, Inc.
|
||||
# Copyright 1992-2013 Free Software Foundation, Inc.
|
||||
|
||||
timestamp='2016-11-04'
|
||||
timestamp='2013-10-01'
|
||||
|
||||
# This file is free software; you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by
|
||||
@@ -25,7 +25,7 @@ timestamp='2016-11-04'
|
||||
# of the GNU General Public License, version 3 ("GPLv3").
|
||||
|
||||
|
||||
# Please send patches to <config-patches@gnu.org>.
|
||||
# Please send patches with a ChangeLog entry to config-patches@gnu.org.
|
||||
#
|
||||
# Configuration subroutine to validate and canonicalize a configuration type.
|
||||
# Supply the specified configuration type as an argument.
|
||||
@@ -33,7 +33,7 @@ timestamp='2016-11-04'
|
||||
# Otherwise, we print the canonical config type on stdout and succeed.
|
||||
|
||||
# You can get the latest version of this script from:
|
||||
# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
|
||||
# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
|
||||
|
||||
# This file is supposed to be the same for all GNU packages
|
||||
# and recognize all the CPU types, system types and aliases
|
||||
@@ -53,7 +53,8 @@ timestamp='2016-11-04'
|
||||
me=`echo "$0" | sed -e 's,.*/,,'`
|
||||
|
||||
usage="\
|
||||
Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS
|
||||
Usage: $0 [OPTION] CPU-MFR-OPSYS
|
||||
$0 [OPTION] ALIAS
|
||||
|
||||
Canonicalize a configuration name.
|
||||
|
||||
@@ -67,7 +68,7 @@ Report bugs and patches to <config-patches@gnu.org>."
|
||||
version="\
|
||||
GNU config.sub ($timestamp)
|
||||
|
||||
Copyright 1992-2016 Free Software Foundation, Inc.
|
||||
Copyright 1992-2013 Free Software Foundation, Inc.
|
||||
|
||||
This is free software; see the source for copying conditions. There is NO
|
||||
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
|
||||
@@ -116,8 +117,8 @@ maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
|
||||
case $maybe_os in
|
||||
nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \
|
||||
linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
|
||||
knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \
|
||||
kopensolaris*-gnu* | cloudabi*-eabi* | \
|
||||
knetbsd*-gnu* | netbsd*-gnu* | \
|
||||
kopensolaris*-gnu* | \
|
||||
storm-chaos* | os2-emx* | rtmk-nova*)
|
||||
os=-$maybe_os
|
||||
basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
|
||||
@@ -254,13 +255,12 @@ case $basic_machine in
|
||||
| arc | arceb \
|
||||
| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \
|
||||
| avr | avr32 \
|
||||
| ba \
|
||||
| be32 | be64 \
|
||||
| bfin \
|
||||
| c4x | c8051 | clipper \
|
||||
| d10v | d30v | dlx | dsp16xx \
|
||||
| e2k | epiphany \
|
||||
| fido | fr30 | frv | ft32 \
|
||||
| epiphany \
|
||||
| fido | fr30 | frv \
|
||||
| h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
|
||||
| hexagon \
|
||||
| i370 | i860 | i960 | ia64 \
|
||||
@@ -283,10 +283,8 @@ case $basic_machine in
|
||||
| mips64vr5900 | mips64vr5900el \
|
||||
| mipsisa32 | mipsisa32el \
|
||||
| mipsisa32r2 | mipsisa32r2el \
|
||||
| mipsisa32r6 | mipsisa32r6el \
|
||||
| mipsisa64 | mipsisa64el \
|
||||
| mipsisa64r2 | mipsisa64r2el \
|
||||
| mipsisa64r6 | mipsisa64r6el \
|
||||
| mipsisa64sb1 | mipsisa64sb1el \
|
||||
| mipsisa64sr71k | mipsisa64sr71kel \
|
||||
| mipsr5900 | mipsr5900el \
|
||||
@@ -298,15 +296,14 @@ case $basic_machine in
|
||||
| nds32 | nds32le | nds32be \
|
||||
| nios | nios2 | nios2eb | nios2el \
|
||||
| ns16k | ns32k \
|
||||
| open8 | or1k | or1knd | or32 \
|
||||
| open8 \
|
||||
| or1k | or32 \
|
||||
| pdp10 | pdp11 | pj | pjl \
|
||||
| powerpc | powerpc64 | powerpc64le | powerpcle \
|
||||
| pru \
|
||||
| pyramid \
|
||||
| riscv32 | riscv64 \
|
||||
| rl78 | rx \
|
||||
| score \
|
||||
| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
|
||||
| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
|
||||
| sh64 | sh64le \
|
||||
| sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \
|
||||
| sparcv8 | sparcv9 | sparcv9b | sparcv9v \
|
||||
@@ -314,7 +311,6 @@ case $basic_machine in
|
||||
| tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \
|
||||
| ubicom32 \
|
||||
| v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
|
||||
| visium \
|
||||
| we32k \
|
||||
| x86 | xc16x | xstormy16 | xtensa \
|
||||
| z8k | z80)
|
||||
@@ -329,9 +325,6 @@ case $basic_machine in
|
||||
c6x)
|
||||
basic_machine=tic6x-unknown
|
||||
;;
|
||||
leon|leon[3-9])
|
||||
basic_machine=sparc-$basic_machine
|
||||
;;
|
||||
m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip)
|
||||
basic_machine=$basic_machine-unknown
|
||||
os=-none
|
||||
@@ -377,13 +370,12 @@ case $basic_machine in
|
||||
| alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \
|
||||
| arm-* | armbe-* | armle-* | armeb-* | armv*-* \
|
||||
| avr-* | avr32-* \
|
||||
| ba-* \
|
||||
| be32-* | be64-* \
|
||||
| bfin-* | bs2000-* \
|
||||
| c[123]* | c30-* | [cjt]90-* | c4x-* \
|
||||
| c8051-* | clipper-* | craynv-* | cydra-* \
|
||||
| d10v-* | d30v-* | dlx-* \
|
||||
| e2k-* | elxsi-* \
|
||||
| elxsi-* \
|
||||
| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \
|
||||
| h8300-* | h8500-* \
|
||||
| hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
|
||||
@@ -410,10 +402,8 @@ case $basic_machine in
|
||||
| mips64vr5900-* | mips64vr5900el-* \
|
||||
| mipsisa32-* | mipsisa32el-* \
|
||||
| mipsisa32r2-* | mipsisa32r2el-* \
|
||||
| mipsisa32r6-* | mipsisa32r6el-* \
|
||||
| mipsisa64-* | mipsisa64el-* \
|
||||
| mipsisa64r2-* | mipsisa64r2el-* \
|
||||
| mipsisa64r6-* | mipsisa64r6el-* \
|
||||
| mipsisa64sb1-* | mipsisa64sb1el-* \
|
||||
| mipsisa64sr71k-* | mipsisa64sr71kel-* \
|
||||
| mipsr5900-* | mipsr5900el-* \
|
||||
@@ -425,19 +415,16 @@ case $basic_machine in
|
||||
| nios-* | nios2-* | nios2eb-* | nios2el-* \
|
||||
| none-* | np1-* | ns16k-* | ns32k-* \
|
||||
| open8-* \
|
||||
| or1k*-* \
|
||||
| orion-* \
|
||||
| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
|
||||
| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \
|
||||
| pru-* \
|
||||
| pyramid-* \
|
||||
| riscv32-* | riscv64-* \
|
||||
| rl78-* | romp-* | rs6000-* | rx-* \
|
||||
| sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \
|
||||
| shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \
|
||||
| sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \
|
||||
| sparclite-* \
|
||||
| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \
|
||||
| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \
|
||||
| tahoe-* \
|
||||
| tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \
|
||||
| tile*-* \
|
||||
@@ -445,7 +432,6 @@ case $basic_machine in
|
||||
| ubicom32-* \
|
||||
| v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
|
||||
| vax-* \
|
||||
| visium-* \
|
||||
| we32k-* \
|
||||
| x86-* | x86_64-* | xc16x-* | xps100-* \
|
||||
| xstormy16-* | xtensa*-* \
|
||||
@@ -522,9 +508,6 @@ case $basic_machine in
|
||||
basic_machine=i386-pc
|
||||
os=-aros
|
||||
;;
|
||||
asmjs)
|
||||
basic_machine=asmjs-unknown
|
||||
;;
|
||||
aux)
|
||||
basic_machine=m68k-apple
|
||||
os=-aux
|
||||
@@ -645,14 +628,6 @@ case $basic_machine in
|
||||
basic_machine=m68k-bull
|
||||
os=-sysv3
|
||||
;;
|
||||
e500v[12])
|
||||
basic_machine=powerpc-unknown
|
||||
os=$os"spe"
|
||||
;;
|
||||
e500v[12]-*)
|
||||
basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
|
||||
os=$os"spe"
|
||||
;;
|
||||
ebmon29k)
|
||||
basic_machine=a29k-amd
|
||||
os=-ebmon
|
||||
@@ -794,9 +769,6 @@ case $basic_machine in
|
||||
basic_machine=m68k-isi
|
||||
os=-sysv
|
||||
;;
|
||||
leon-*|leon[3-9]-*)
|
||||
basic_machine=sparc-`echo $basic_machine | sed 's/-.*//'`
|
||||
;;
|
||||
m68knommu)
|
||||
basic_machine=m68k-unknown
|
||||
os=-linux
|
||||
@@ -852,10 +824,6 @@ case $basic_machine in
|
||||
basic_machine=powerpc-unknown
|
||||
os=-morphos
|
||||
;;
|
||||
moxiebox)
|
||||
basic_machine=moxie-unknown
|
||||
os=-moxiebox
|
||||
;;
|
||||
msdos)
|
||||
basic_machine=i386-pc
|
||||
os=-msdos
|
||||
@@ -1032,7 +1000,7 @@ case $basic_machine in
|
||||
ppc-* | ppcbe-*)
|
||||
basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
|
||||
;;
|
||||
ppcle | powerpclittle)
|
||||
ppcle | powerpclittle | ppc-le | powerpc-little)
|
||||
basic_machine=powerpcle-unknown
|
||||
;;
|
||||
ppcle-* | powerpclittle-*)
|
||||
@@ -1042,7 +1010,7 @@ case $basic_machine in
|
||||
;;
|
||||
ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
|
||||
;;
|
||||
ppc64le | powerpc64little)
|
||||
ppc64le | powerpc64little | ppc64-le | powerpc64-little)
|
||||
basic_machine=powerpc64le-unknown
|
||||
;;
|
||||
ppc64le-* | powerpc64little-*)
|
||||
@@ -1388,28 +1356,27 @@ case $os in
|
||||
| -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
|
||||
| -sym* | -kopensolaris* | -plan9* \
|
||||
| -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
|
||||
| -aos* | -aros* | -cloudabi* | -sortix* \
|
||||
| -aos* | -aros* \
|
||||
| -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
|
||||
| -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
|
||||
| -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \
|
||||
| -bitrig* | -openbsd* | -solidbsd* | -libertybsd* \
|
||||
| -bitrig* | -openbsd* | -solidbsd* \
|
||||
| -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \
|
||||
| -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
|
||||
| -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
|
||||
| -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
|
||||
| -chorusos* | -chorusrdb* | -cegcc* \
|
||||
| -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
|
||||
| -midipix* | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \
|
||||
| -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \
|
||||
| -linux-newlib* | -linux-musl* | -linux-uclibc* \
|
||||
| -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \
|
||||
| -uxpv* | -beos* | -mpeix* | -udk* \
|
||||
| -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \
|
||||
| -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
|
||||
| -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \
|
||||
| -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \
|
||||
| -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
|
||||
| -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
|
||||
| -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \
|
||||
| -onefs* | -tirtos* | -phoenix* | -fuchsia*)
|
||||
| -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*)
|
||||
# Remember, each alternative MUST END IN *, to match a version number.
|
||||
;;
|
||||
-qnx*)
|
||||
@@ -1541,8 +1508,6 @@ case $os in
|
||||
;;
|
||||
-nacl*)
|
||||
;;
|
||||
-ios)
|
||||
;;
|
||||
-none)
|
||||
;;
|
||||
*)
|
||||
@@ -1629,6 +1594,9 @@ case $basic_machine in
|
||||
mips*-*)
|
||||
os=-elf
|
||||
;;
|
||||
or1k-*)
|
||||
os=-elf
|
||||
;;
|
||||
or32-*)
|
||||
os=-coff
|
||||
;;
|
||||
934
configure.ac
934
configure.ac
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,4 @@
|
||||
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
|
||||
<xsl:import href="@XSLROOT@/html/docbook.xsl"/>
|
||||
<xsl:import href="@abs_srcroot@doc/stylesheet.xsl"/>
|
||||
<xsl:output method="xml" encoding="utf-8"/>
|
||||
</xsl:stylesheet>
|
||||
|
||||
1645
doc/jemalloc.xml.in
1645
doc/jemalloc.xml.in
File diff suppressed because it is too large
Load Diff
@@ -1,10 +1,7 @@
|
||||
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
|
||||
<xsl:param name="funcsynopsis.style">ansi</xsl:param>
|
||||
<xsl:param name="function.parens" select="0"/>
|
||||
<xsl:template match="function">
|
||||
<xsl:call-template name="inline.monoseq"/>
|
||||
</xsl:template>
|
||||
<xsl:param name="function.parens" select="1"/>
|
||||
<xsl:template match="mallctl">
|
||||
<quote><xsl:call-template name="inline.monoseq"/></quote>
|
||||
"<xsl:call-template name="inline.monoseq"/>"
|
||||
</xsl:template>
|
||||
</xsl:stylesheet>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,45 +0,0 @@
|
||||
/*
|
||||
* Define a custom assert() in order to reduce the chances of deadlock during
|
||||
* assertion failure.
|
||||
*/
|
||||
#ifndef assert
|
||||
#define assert(e) do { \
|
||||
if (unlikely(config_debug && !(e))) { \
|
||||
malloc_printf( \
|
||||
"<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", \
|
||||
__FILE__, __LINE__, #e); \
|
||||
abort(); \
|
||||
} \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#ifndef not_reached
|
||||
#define not_reached() do { \
|
||||
if (config_debug) { \
|
||||
malloc_printf( \
|
||||
"<jemalloc>: %s:%d: Unreachable code reached\n", \
|
||||
__FILE__, __LINE__); \
|
||||
abort(); \
|
||||
} \
|
||||
unreachable(); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#ifndef not_implemented
|
||||
#define not_implemented() do { \
|
||||
if (config_debug) { \
|
||||
malloc_printf("<jemalloc>: %s:%d: Not implemented\n", \
|
||||
__FILE__, __LINE__); \
|
||||
abort(); \
|
||||
} \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#ifndef assert_not_implemented
|
||||
#define assert_not_implemented(e) do { \
|
||||
if (unlikely(config_debug && !(e))) \
|
||||
not_implemented(); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
|
||||
#define atomic_read_uint64(p) atomic_add_uint64(p, 0)
|
||||
#define atomic_read_uint32(p) atomic_add_uint32(p, 0)
|
||||
#define atomic_read_p(p) atomic_add_p(p, NULL)
|
||||
#define atomic_read_z(p) atomic_add_z(p, 0)
|
||||
#define atomic_read_u(p) atomic_add_u(p, 0)
|
||||
|
||||
@@ -19,139 +18,89 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
/*
|
||||
* All arithmetic functions return the arithmetic result of the atomic
|
||||
* operation. Some atomic operation APIs return the value prior to mutation, in
|
||||
* which case the following functions must redundantly compute the result so
|
||||
* that it can be returned. These functions are normally inlined, so the extra
|
||||
* operations can be optimized away if the return values aren't used by the
|
||||
* callers.
|
||||
*
|
||||
* <t> atomic_read_<t>(<t> *p) { return (*p); }
|
||||
* <t> atomic_add_<t>(<t> *p, <t> x) { return (*p += x); }
|
||||
* <t> atomic_sub_<t>(<t> *p, <t> x) { return (*p -= x); }
|
||||
* bool atomic_cas_<t>(<t> *p, <t> c, <t> s)
|
||||
* {
|
||||
* if (*p != c)
|
||||
* return (true);
|
||||
* *p = s;
|
||||
* return (false);
|
||||
* }
|
||||
* void atomic_write_<t>(<t> *p, <t> x) { *p = x; }
|
||||
*/
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
uint64_t atomic_add_uint64(uint64_t *p, uint64_t x);
|
||||
uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x);
|
||||
bool atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s);
|
||||
void atomic_write_uint64(uint64_t *p, uint64_t x);
|
||||
uint32_t atomic_add_uint32(uint32_t *p, uint32_t x);
|
||||
uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x);
|
||||
bool atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s);
|
||||
void atomic_write_uint32(uint32_t *p, uint32_t x);
|
||||
void *atomic_add_p(void **p, void *x);
|
||||
void *atomic_sub_p(void **p, void *x);
|
||||
bool atomic_cas_p(void **p, void *c, void *s);
|
||||
void atomic_write_p(void **p, const void *x);
|
||||
size_t atomic_add_z(size_t *p, size_t x);
|
||||
size_t atomic_sub_z(size_t *p, size_t x);
|
||||
bool atomic_cas_z(size_t *p, size_t c, size_t s);
|
||||
void atomic_write_z(size_t *p, size_t x);
|
||||
unsigned atomic_add_u(unsigned *p, unsigned x);
|
||||
unsigned atomic_sub_u(unsigned *p, unsigned x);
|
||||
bool atomic_cas_u(unsigned *p, unsigned c, unsigned s);
|
||||
void atomic_write_u(unsigned *p, unsigned x);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))
|
||||
/******************************************************************************/
|
||||
/* 64-bit operations. */
|
||||
#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
|
||||
# if (defined(__amd64__) || defined(__x86_64__))
|
||||
# ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_add_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
uint64_t t = x;
|
||||
|
||||
asm volatile (
|
||||
"lock; xaddq %0, %1;"
|
||||
: "+r" (t), "=m" (*p) /* Outputs. */
|
||||
: "m" (*p) /* Inputs. */
|
||||
);
|
||||
|
||||
return (t + x);
|
||||
return (__sync_add_and_fetch(p, x));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_sub_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
return (__sync_sub_and_fetch(p, x));
|
||||
}
|
||||
#elif (defined(_MSC_VER))
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_add_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
return (InterlockedExchangeAdd64(p, x));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_sub_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
return (InterlockedExchangeAdd64(p, -((int64_t)x)));
|
||||
}
|
||||
#elif (defined(JEMALLOC_OSATOMIC))
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_add_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
return (OSAtomicAdd64((int64_t)x, (int64_t *)p));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_sub_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
|
||||
}
|
||||
# elif (defined(__amd64__) || defined(__x86_64__))
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_add_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
asm volatile (
|
||||
"lock; xaddq %0, %1;"
|
||||
: "+r" (x), "=m" (*p) /* Outputs. */
|
||||
: "m" (*p) /* Inputs. */
|
||||
);
|
||||
|
||||
return (x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_sub_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
uint64_t t;
|
||||
|
||||
x = (uint64_t)(-(int64_t)x);
|
||||
t = x;
|
||||
asm volatile (
|
||||
"lock; xaddq %0, %1;"
|
||||
: "+r" (t), "=m" (*p) /* Outputs. */
|
||||
: "+r" (x), "=m" (*p) /* Outputs. */
|
||||
: "m" (*p) /* Inputs. */
|
||||
);
|
||||
|
||||
return (t + x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
|
||||
{
|
||||
uint8_t success;
|
||||
|
||||
asm volatile (
|
||||
"lock; cmpxchgq %4, %0;"
|
||||
"sete %1;"
|
||||
: "=m" (*p), "=a" (success) /* Outputs. */
|
||||
: "m" (*p), "a" (c), "r" (s) /* Inputs. */
|
||||
: "memory" /* Clobbers. */
|
||||
);
|
||||
|
||||
return (!(bool)success);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
asm volatile (
|
||||
"xchgq %1, %0;" /* Lock is implied by xchgq. */
|
||||
: "=m" (*p), "+r" (x) /* Outputs. */
|
||||
: "m" (*p) /* Inputs. */
|
||||
: "memory" /* Clobbers. */
|
||||
);
|
||||
}
|
||||
# elif (defined(JEMALLOC_C11ATOMICS))
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_add_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
|
||||
return (atomic_fetch_add(a, x) + x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_sub_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
|
||||
return (atomic_fetch_sub(a, x) - x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
|
||||
{
|
||||
volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
|
||||
return (!atomic_compare_exchange_strong(a, &c, s));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
|
||||
atomic_store(a, x);
|
||||
return (x);
|
||||
}
|
||||
# elif (defined(JEMALLOC_ATOMIC9))
|
||||
JEMALLOC_INLINE uint64_t
|
||||
@@ -175,88 +124,7 @@ atomic_sub_uint64(uint64_t *p, uint64_t x)
|
||||
|
||||
return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
|
||||
{
|
||||
|
||||
assert(sizeof(uint64_t) == sizeof(unsigned long));
|
||||
|
||||
return (!atomic_cmpset_long(p, (unsigned long)c, (unsigned long)s));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
assert(sizeof(uint64_t) == sizeof(unsigned long));
|
||||
|
||||
atomic_store_rel_long(p, x);
|
||||
}
|
||||
# elif (defined(JEMALLOC_OSATOMIC))
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_add_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
return (OSAtomicAdd64((int64_t)x, (int64_t *)p));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_sub_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
|
||||
{
|
||||
|
||||
return (!OSAtomicCompareAndSwap64(c, s, (int64_t *)p));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
uint64_t o;
|
||||
|
||||
/*The documented OSAtomic*() API does not expose an atomic exchange. */
|
||||
do {
|
||||
o = atomic_read_uint64(p);
|
||||
} while (atomic_cas_uint64(p, o, x));
|
||||
}
|
||||
# elif (defined(_MSC_VER))
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_add_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
return (InterlockedExchangeAdd64(p, x) + x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_sub_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
return (InterlockedExchangeAdd64(p, -((int64_t)x)) - x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
|
||||
{
|
||||
uint64_t o;
|
||||
|
||||
o = InterlockedCompareExchange64(p, s, c);
|
||||
return (o != c);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
InterlockedExchange64(p, x);
|
||||
}
|
||||
# elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) || \
|
||||
defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
|
||||
# elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
|
||||
JEMALLOC_INLINE uint64_t
|
||||
atomic_add_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
@@ -270,20 +138,6 @@ atomic_sub_uint64(uint64_t *p, uint64_t x)
|
||||
|
||||
return (__sync_sub_and_fetch(p, x));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
|
||||
{
|
||||
|
||||
return (!__sync_bool_compare_and_swap(p, c, s));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_uint64(uint64_t *p, uint64_t x)
|
||||
{
|
||||
|
||||
__sync_lock_test_and_set(p, x);
|
||||
}
|
||||
# else
|
||||
# error "Missing implementation for 64-bit atomic operations"
|
||||
# endif
|
||||
@@ -291,119 +145,33 @@ atomic_write_uint64(uint64_t *p, uint64_t x)
|
||||
|
||||
/******************************************************************************/
|
||||
/* 32-bit operations. */
|
||||
#if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_add_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
uint32_t t = x;
|
||||
|
||||
asm volatile (
|
||||
"lock; xaddl %0, %1;"
|
||||
: "+r" (t), "=m" (*p) /* Outputs. */
|
||||
: "m" (*p) /* Inputs. */
|
||||
);
|
||||
|
||||
return (t + x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_sub_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
uint32_t t;
|
||||
|
||||
x = (uint32_t)(-(int32_t)x);
|
||||
t = x;
|
||||
asm volatile (
|
||||
"lock; xaddl %0, %1;"
|
||||
: "+r" (t), "=m" (*p) /* Outputs. */
|
||||
: "m" (*p) /* Inputs. */
|
||||
);
|
||||
|
||||
return (t + x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
|
||||
{
|
||||
uint8_t success;
|
||||
|
||||
asm volatile (
|
||||
"lock; cmpxchgl %4, %0;"
|
||||
"sete %1;"
|
||||
: "=m" (*p), "=a" (success) /* Outputs. */
|
||||
: "m" (*p), "a" (c), "r" (s) /* Inputs. */
|
||||
: "memory"
|
||||
);
|
||||
|
||||
return (!(bool)success);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
|
||||
asm volatile (
|
||||
"xchgl %1, %0;" /* Lock is implied by xchgl. */
|
||||
: "=m" (*p), "+r" (x) /* Outputs. */
|
||||
: "m" (*p) /* Inputs. */
|
||||
: "memory" /* Clobbers. */
|
||||
);
|
||||
}
|
||||
# elif (defined(JEMALLOC_C11ATOMICS))
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_add_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
|
||||
return (atomic_fetch_add(a, x) + x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_sub_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
|
||||
return (atomic_fetch_sub(a, x) - x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
|
||||
{
|
||||
volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
|
||||
return (!atomic_compare_exchange_strong(a, &c, s));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
|
||||
atomic_store(a, x);
|
||||
}
|
||||
#elif (defined(JEMALLOC_ATOMIC9))
|
||||
#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_add_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
|
||||
return (atomic_fetchadd_32(p, x) + x);
|
||||
return (__sync_add_and_fetch(p, x));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_sub_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
|
||||
return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);
|
||||
return (__sync_sub_and_fetch(p, x));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
|
||||
#elif (defined(_MSC_VER))
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_add_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
|
||||
return (!atomic_cmpset_32(p, c, s));
|
||||
return (InterlockedExchangeAdd(p, x));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_uint32(uint32_t *p, uint32_t x)
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_sub_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
|
||||
atomic_store_rel_32(p, x);
|
||||
return (InterlockedExchangeAdd(p, -((int32_t)x)));
|
||||
}
|
||||
#elif (defined(JEMALLOC_OSATOMIC))
|
||||
JEMALLOC_INLINE uint32_t
|
||||
@@ -419,56 +187,48 @@ atomic_sub_uint32(uint32_t *p, uint32_t x)
|
||||
|
||||
return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
|
||||
{
|
||||
|
||||
return (!OSAtomicCompareAndSwap32(c, s, (int32_t *)p));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
uint32_t o;
|
||||
|
||||
/*The documented OSAtomic*() API does not expose an atomic exchange. */
|
||||
do {
|
||||
o = atomic_read_uint32(p);
|
||||
} while (atomic_cas_uint32(p, o, x));
|
||||
}
|
||||
#elif (defined(_MSC_VER))
|
||||
#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_add_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
|
||||
return (InterlockedExchangeAdd(p, x) + x);
|
||||
asm volatile (
|
||||
"lock; xaddl %0, %1;"
|
||||
: "+r" (x), "=m" (*p) /* Outputs. */
|
||||
: "m" (*p) /* Inputs. */
|
||||
);
|
||||
|
||||
return (x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_sub_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
|
||||
return (InterlockedExchangeAdd(p, -((int32_t)x)) - x);
|
||||
x = (uint32_t)(-(int32_t)x);
|
||||
asm volatile (
|
||||
"lock; xaddl %0, %1;"
|
||||
: "+r" (x), "=m" (*p) /* Outputs. */
|
||||
: "m" (*p) /* Inputs. */
|
||||
);
|
||||
|
||||
return (x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
|
||||
{
|
||||
uint32_t o;
|
||||
|
||||
o = InterlockedCompareExchange(p, s, c);
|
||||
return (o != c);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_uint32(uint32_t *p, uint32_t x)
|
||||
#elif (defined(JEMALLOC_ATOMIC9))
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_add_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
|
||||
InterlockedExchange(p, x);
|
||||
return (atomic_fetchadd_32(p, x) + x);
|
||||
}
|
||||
#elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) || \
|
||||
defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
|
||||
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_sub_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
|
||||
return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);
|
||||
}
|
||||
#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
|
||||
JEMALLOC_INLINE uint32_t
|
||||
atomic_add_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
@@ -482,72 +242,10 @@ atomic_sub_uint32(uint32_t *p, uint32_t x)
|
||||
|
||||
return (__sync_sub_and_fetch(p, x));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
|
||||
{
|
||||
|
||||
return (!__sync_bool_compare_and_swap(p, c, s));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_uint32(uint32_t *p, uint32_t x)
|
||||
{
|
||||
|
||||
__sync_lock_test_and_set(p, x);
|
||||
}
|
||||
#else
|
||||
# error "Missing implementation for 32-bit atomic operations"
|
||||
#endif
|
||||
|
||||
/******************************************************************************/
|
||||
/* Pointer operations. */
|
||||
JEMALLOC_INLINE void *
|
||||
atomic_add_p(void **p, void *x)
|
||||
{
|
||||
|
||||
#if (LG_SIZEOF_PTR == 3)
|
||||
return ((void *)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
|
||||
#elif (LG_SIZEOF_PTR == 2)
|
||||
return ((void *)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
|
||||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void *
|
||||
atomic_sub_p(void **p, void *x)
|
||||
{
|
||||
|
||||
#if (LG_SIZEOF_PTR == 3)
|
||||
return ((void *)atomic_add_uint64((uint64_t *)p,
|
||||
(uint64_t)-((int64_t)x)));
|
||||
#elif (LG_SIZEOF_PTR == 2)
|
||||
return ((void *)atomic_add_uint32((uint32_t *)p,
|
||||
(uint32_t)-((int32_t)x)));
|
||||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_p(void **p, void *c, void *s)
|
||||
{
|
||||
|
||||
#if (LG_SIZEOF_PTR == 3)
|
||||
return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s));
|
||||
#elif (LG_SIZEOF_PTR == 2)
|
||||
return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s));
|
||||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_p(void **p, const void *x)
|
||||
{
|
||||
|
||||
#if (LG_SIZEOF_PTR == 3)
|
||||
atomic_write_uint64((uint64_t *)p, (uint64_t)x);
|
||||
#elif (LG_SIZEOF_PTR == 2)
|
||||
atomic_write_uint32((uint32_t *)p, (uint32_t)x);
|
||||
#endif
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
/* size_t operations. */
|
||||
JEMALLOC_INLINE size_t
|
||||
@@ -574,28 +272,6 @@ atomic_sub_z(size_t *p, size_t x)
|
||||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_z(size_t *p, size_t c, size_t s)
|
||||
{
|
||||
|
||||
#if (LG_SIZEOF_PTR == 3)
|
||||
return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s));
|
||||
#elif (LG_SIZEOF_PTR == 2)
|
||||
return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s));
|
||||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_z(size_t *p, size_t x)
|
||||
{
|
||||
|
||||
#if (LG_SIZEOF_PTR == 3)
|
||||
atomic_write_uint64((uint64_t *)p, (uint64_t)x);
|
||||
#elif (LG_SIZEOF_PTR == 2)
|
||||
atomic_write_uint32((uint32_t *)p, (uint32_t)x);
|
||||
#endif
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
/* unsigned operations. */
|
||||
JEMALLOC_INLINE unsigned
|
||||
@@ -621,29 +297,6 @@ atomic_sub_u(unsigned *p, unsigned x)
|
||||
(uint32_t)-((int32_t)x)));
|
||||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
atomic_cas_u(unsigned *p, unsigned c, unsigned s)
|
||||
{
|
||||
|
||||
#if (LG_SIZEOF_INT == 3)
|
||||
return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s));
|
||||
#elif (LG_SIZEOF_INT == 2)
|
||||
return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s));
|
||||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
atomic_write_u(unsigned *p, unsigned x)
|
||||
{
|
||||
|
||||
#if (LG_SIZEOF_INT == 3)
|
||||
atomic_write_uint64((uint64_t *)p, (uint64_t)x);
|
||||
#elif (LG_SIZEOF_INT == 2)
|
||||
atomic_write_uint32((uint32_t *)p, (uint32_t)x);
|
||||
#endif
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
#endif
|
||||
|
||||
|
||||
@@ -9,13 +9,14 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
void *base_alloc(tsdn_t *tsdn, size_t size);
|
||||
void base_stats_get(tsdn_t *tsdn, size_t *allocated, size_t *resident,
|
||||
size_t *mapped);
|
||||
void *base_alloc(size_t size);
|
||||
void *base_calloc(size_t number, size_t size);
|
||||
extent_node_t *base_node_alloc(void);
|
||||
void base_node_dealloc(extent_node_t *node);
|
||||
bool base_boot(void);
|
||||
void base_prefork(tsdn_t *tsdn);
|
||||
void base_postfork_parent(tsdn_t *tsdn);
|
||||
void base_postfork_child(tsdn_t *tsdn);
|
||||
void base_prefork(void);
|
||||
void base_postfork_parent(void);
|
||||
void base_postfork_child(void);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
/* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */
|
||||
#define LG_BITMAP_MAXBITS LG_RUN_MAXREGS
|
||||
#define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS)
|
||||
|
||||
typedef struct bitmap_level_s bitmap_level_t;
|
||||
typedef struct bitmap_info_s bitmap_info_t;
|
||||
@@ -15,73 +14,11 @@ typedef unsigned long bitmap_t;
|
||||
#define BITMAP_GROUP_NBITS (ZU(1) << LG_BITMAP_GROUP_NBITS)
|
||||
#define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS-1)
|
||||
|
||||
/*
|
||||
* Do some analysis on how big the bitmap is before we use a tree. For a brute
|
||||
* force linear search, if we would have to call ffs_lu() more than 2^3 times,
|
||||
* use a tree instead.
|
||||
*/
|
||||
#if LG_BITMAP_MAXBITS - LG_BITMAP_GROUP_NBITS > 3
|
||||
# define USE_TREE
|
||||
#endif
|
||||
|
||||
/* Number of groups required to store a given number of bits. */
|
||||
#define BITMAP_BITS2GROUPS(nbits) \
|
||||
((nbits + BITMAP_GROUP_NBITS_MASK) >> LG_BITMAP_GROUP_NBITS)
|
||||
|
||||
/*
|
||||
* Number of groups required at a particular level for a given number of bits.
|
||||
*/
|
||||
#define BITMAP_GROUPS_L0(nbits) \
|
||||
BITMAP_BITS2GROUPS(nbits)
|
||||
#define BITMAP_GROUPS_L1(nbits) \
|
||||
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(nbits))
|
||||
#define BITMAP_GROUPS_L2(nbits) \
|
||||
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits))))
|
||||
#define BITMAP_GROUPS_L3(nbits) \
|
||||
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS( \
|
||||
BITMAP_BITS2GROUPS((nbits)))))
|
||||
|
||||
/*
|
||||
* Assuming the number of levels, number of groups required for a given number
|
||||
* of bits.
|
||||
*/
|
||||
#define BITMAP_GROUPS_1_LEVEL(nbits) \
|
||||
BITMAP_GROUPS_L0(nbits)
|
||||
#define BITMAP_GROUPS_2_LEVEL(nbits) \
|
||||
(BITMAP_GROUPS_1_LEVEL(nbits) + BITMAP_GROUPS_L1(nbits))
|
||||
#define BITMAP_GROUPS_3_LEVEL(nbits) \
|
||||
(BITMAP_GROUPS_2_LEVEL(nbits) + BITMAP_GROUPS_L2(nbits))
|
||||
#define BITMAP_GROUPS_4_LEVEL(nbits) \
|
||||
(BITMAP_GROUPS_3_LEVEL(nbits) + BITMAP_GROUPS_L3(nbits))
|
||||
|
||||
/*
|
||||
* Maximum number of groups required to support LG_BITMAP_MAXBITS.
|
||||
*/
|
||||
#ifdef USE_TREE
|
||||
|
||||
#if LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS
|
||||
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_1_LEVEL(BITMAP_MAXBITS)
|
||||
#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 2
|
||||
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_2_LEVEL(BITMAP_MAXBITS)
|
||||
#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 3
|
||||
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_3_LEVEL(BITMAP_MAXBITS)
|
||||
#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 4
|
||||
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_4_LEVEL(BITMAP_MAXBITS)
|
||||
#else
|
||||
# error "Unsupported bitmap size"
|
||||
#endif
|
||||
|
||||
/* Maximum number of levels possible. */
|
||||
#define BITMAP_MAX_LEVELS \
|
||||
(LG_BITMAP_MAXBITS / LG_SIZEOF_BITMAP) \
|
||||
+ !!(LG_BITMAP_MAXBITS % LG_SIZEOF_BITMAP)
|
||||
|
||||
#else /* USE_TREE */
|
||||
|
||||
#define BITMAP_GROUPS_MAX BITMAP_BITS2GROUPS(BITMAP_MAXBITS)
|
||||
|
||||
#endif /* USE_TREE */
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
@@ -95,7 +32,6 @@ struct bitmap_info_s {
|
||||
/* Logical number of bits in bitmap (stored at bottom level). */
|
||||
size_t nbits;
|
||||
|
||||
#ifdef USE_TREE
|
||||
/* Number of levels necessary for nbits. */
|
||||
unsigned nlevels;
|
||||
|
||||
@@ -104,10 +40,6 @@ struct bitmap_info_s {
|
||||
* bottom to top (e.g. the bottom level is stored in levels[0]).
|
||||
*/
|
||||
bitmap_level_t levels[BITMAP_MAX_LEVELS+1];
|
||||
#else /* USE_TREE */
|
||||
/* Number of groups necessary for nbits. */
|
||||
size_t ngroups;
|
||||
#endif /* USE_TREE */
|
||||
};
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
@@ -115,8 +47,9 @@ struct bitmap_info_s {
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
void bitmap_info_init(bitmap_info_t *binfo, size_t nbits);
|
||||
size_t bitmap_info_ngroups(const bitmap_info_t *binfo);
|
||||
size_t bitmap_size(size_t nbits);
|
||||
void bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo);
|
||||
size_t bitmap_size(const bitmap_info_t *binfo);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
@@ -134,20 +67,10 @@ void bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);
|
||||
JEMALLOC_INLINE bool
|
||||
bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo)
|
||||
{
|
||||
#ifdef USE_TREE
|
||||
size_t rgoff = binfo->levels[binfo->nlevels].group_offset - 1;
|
||||
unsigned rgoff = binfo->levels[binfo->nlevels].group_offset - 1;
|
||||
bitmap_t rg = bitmap[rgoff];
|
||||
/* The bitmap is full iff the root group is 0. */
|
||||
return (rg == 0);
|
||||
#else
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < binfo->ngroups; i++) {
|
||||
if (bitmap[i] != 0)
|
||||
return (false);
|
||||
}
|
||||
return (true);
|
||||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
@@ -159,7 +82,7 @@ bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
|
||||
assert(bit < binfo->nbits);
|
||||
goff = bit >> LG_BITMAP_GROUP_NBITS;
|
||||
g = bitmap[goff];
|
||||
return (!(g & (ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK))));
|
||||
return (!(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
@@ -170,15 +93,14 @@ bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
|
||||
bitmap_t g;
|
||||
|
||||
assert(bit < binfo->nbits);
|
||||
assert(!bitmap_get(bitmap, binfo, bit));
|
||||
assert(bitmap_get(bitmap, binfo, bit) == false);
|
||||
goff = bit >> LG_BITMAP_GROUP_NBITS;
|
||||
gp = &bitmap[goff];
|
||||
g = *gp;
|
||||
assert(g & (ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK)));
|
||||
g ^= ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK);
|
||||
assert(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)));
|
||||
g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);
|
||||
*gp = g;
|
||||
assert(bitmap_get(bitmap, binfo, bit));
|
||||
#ifdef USE_TREE
|
||||
/* Propagate group state transitions up the tree. */
|
||||
if (g == 0) {
|
||||
unsigned i;
|
||||
@@ -187,14 +109,13 @@ bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
|
||||
goff = bit >> LG_BITMAP_GROUP_NBITS;
|
||||
gp = &bitmap[binfo->levels[i].group_offset + goff];
|
||||
g = *gp;
|
||||
assert(g & (ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK)));
|
||||
g ^= ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK);
|
||||
assert(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)));
|
||||
g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);
|
||||
*gp = g;
|
||||
if (g != 0)
|
||||
break;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/* sfu: set first unset. */
|
||||
@@ -205,26 +126,17 @@ bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo)
|
||||
bitmap_t g;
|
||||
unsigned i;
|
||||
|
||||
assert(!bitmap_full(bitmap, binfo));
|
||||
assert(bitmap_full(bitmap, binfo) == false);
|
||||
|
||||
#ifdef USE_TREE
|
||||
i = binfo->nlevels - 1;
|
||||
g = bitmap[binfo->levels[i].group_offset];
|
||||
bit = ffs_lu(g) - 1;
|
||||
bit = ffsl(g) - 1;
|
||||
while (i > 0) {
|
||||
i--;
|
||||
g = bitmap[binfo->levels[i].group_offset + bit];
|
||||
bit = (bit << LG_BITMAP_GROUP_NBITS) + (ffs_lu(g) - 1);
|
||||
bit = (bit << LG_BITMAP_GROUP_NBITS) + (ffsl(g) - 1);
|
||||
}
|
||||
#else
|
||||
i = 0;
|
||||
g = bitmap[0];
|
||||
while ((bit = ffs_lu(g)) == 0) {
|
||||
i++;
|
||||
g = bitmap[i];
|
||||
}
|
||||
bit = (i << LG_BITMAP_GROUP_NBITS) + (bit - 1);
|
||||
#endif
|
||||
|
||||
bitmap_set(bitmap, binfo, bit);
|
||||
return (bit);
|
||||
}
|
||||
@@ -235,7 +147,7 @@ bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
|
||||
size_t goff;
|
||||
bitmap_t *gp;
|
||||
bitmap_t g;
|
||||
UNUSED bool propagate;
|
||||
bool propagate;
|
||||
|
||||
assert(bit < binfo->nbits);
|
||||
assert(bitmap_get(bitmap, binfo, bit));
|
||||
@@ -243,11 +155,10 @@ bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
|
||||
gp = &bitmap[goff];
|
||||
g = *gp;
|
||||
propagate = (g == 0);
|
||||
assert((g & (ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK))) == 0);
|
||||
g ^= ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK);
|
||||
assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))) == 0);
|
||||
g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);
|
||||
*gp = g;
|
||||
assert(!bitmap_get(bitmap, binfo, bit));
|
||||
#ifdef USE_TREE
|
||||
assert(bitmap_get(bitmap, binfo, bit) == false);
|
||||
/* Propagate group state transitions up the tree. */
|
||||
if (propagate) {
|
||||
unsigned i;
|
||||
@@ -257,15 +168,14 @@ bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
|
||||
gp = &bitmap[binfo->levels[i].group_offset + goff];
|
||||
g = *gp;
|
||||
propagate = (g == 0);
|
||||
assert((g & (ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK)))
|
||||
assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)))
|
||||
== 0);
|
||||
g ^= ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK);
|
||||
g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);
|
||||
*gp = g;
|
||||
if (!propagate)
|
||||
if (propagate == false)
|
||||
break;
|
||||
}
|
||||
}
|
||||
#endif /* USE_TREE */
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* Size and alignment of memory chunks that are allocated by the OS's virtual
|
||||
* memory system.
|
||||
*/
|
||||
#define LG_CHUNK_DEFAULT 21
|
||||
#define LG_CHUNK_DEFAULT 22
|
||||
|
||||
/* Return the chunk address for allocation address a. */
|
||||
#define CHUNK_ADDR2BASE(a) \
|
||||
@@ -19,16 +19,6 @@
|
||||
#define CHUNK_CEILING(s) \
|
||||
(((s) + chunksize_mask) & ~chunksize_mask)
|
||||
|
||||
#define CHUNK_HOOKS_INITIALIZER { \
|
||||
NULL, \
|
||||
NULL, \
|
||||
NULL, \
|
||||
NULL, \
|
||||
NULL, \
|
||||
NULL, \
|
||||
NULL \
|
||||
}
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
@@ -40,56 +30,32 @@
|
||||
extern size_t opt_lg_chunk;
|
||||
extern const char *opt_dss;
|
||||
|
||||
extern rtree_t chunks_rtree;
|
||||
/* Protects stats_chunks; currently not used for any other purpose. */
|
||||
extern malloc_mutex_t chunks_mtx;
|
||||
/* Chunk statistics. */
|
||||
extern chunk_stats_t stats_chunks;
|
||||
|
||||
extern rtree_t *chunks_rtree;
|
||||
|
||||
extern size_t chunksize;
|
||||
extern size_t chunksize_mask; /* (chunksize - 1). */
|
||||
extern size_t chunk_npages;
|
||||
extern size_t map_bias; /* Number of arena chunk header pages. */
|
||||
extern size_t arena_maxclass; /* Max size class for arenas. */
|
||||
|
||||
extern const chunk_hooks_t chunk_hooks_default;
|
||||
|
||||
chunk_hooks_t chunk_hooks_get(tsdn_t *tsdn, arena_t *arena);
|
||||
chunk_hooks_t chunk_hooks_set(tsdn_t *tsdn, arena_t *arena,
|
||||
const chunk_hooks_t *chunk_hooks);
|
||||
|
||||
bool chunk_register(tsdn_t *tsdn, const void *chunk,
|
||||
const extent_node_t *node);
|
||||
void chunk_deregister(const void *chunk, const extent_node_t *node);
|
||||
void *chunk_alloc_base(size_t size);
|
||||
void *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena,
|
||||
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
|
||||
size_t *sn, bool *zero, bool *commit, bool dalloc_node);
|
||||
void *chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
|
||||
size_t *sn, bool *zero, bool *commit);
|
||||
void chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
|
||||
chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t sn,
|
||||
bool committed);
|
||||
void chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t sn,
|
||||
bool zeroed, bool committed);
|
||||
bool chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t offset,
|
||||
size_t length);
|
||||
void *chunk_alloc(size_t size, size_t alignment, bool base, bool *zero,
|
||||
dss_prec_t dss_prec);
|
||||
void chunk_unmap(void *chunk, size_t size);
|
||||
void chunk_dealloc(void *chunk, size_t size, bool unmap);
|
||||
bool chunk_boot(void);
|
||||
void chunk_prefork(void);
|
||||
void chunk_postfork_parent(void);
|
||||
void chunk_postfork_child(void);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
extent_node_t *chunk_lookup(const void *chunk, bool dependent);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_CHUNK_C_))
|
||||
JEMALLOC_INLINE extent_node_t *
|
||||
chunk_lookup(const void *ptr, bool dependent)
|
||||
{
|
||||
|
||||
return (rtree_get(&chunks_rtree, (uintptr_t)ptr, dependent));
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
|
||||
|
||||
@@ -23,11 +23,12 @@ extern const char *dss_prec_names[];
|
||||
|
||||
dss_prec_t chunk_dss_prec_get(void);
|
||||
bool chunk_dss_prec_set(dss_prec_t dss_prec);
|
||||
void *chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr,
|
||||
size_t size, size_t alignment, bool *zero, bool *commit);
|
||||
void *chunk_alloc_dss(size_t size, size_t alignment, bool *zero);
|
||||
bool chunk_in_dss(void *chunk);
|
||||
bool chunk_dss_mergeable(void *chunk_a, void *chunk_b);
|
||||
void chunk_dss_boot(void);
|
||||
bool chunk_dss_boot(void);
|
||||
void chunk_dss_prefork(void);
|
||||
void chunk_dss_postfork_parent(void);
|
||||
void chunk_dss_postfork_child(void);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
|
||||
@@ -9,9 +9,10 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
void *chunk_alloc_mmap(void *new_addr, size_t size, size_t alignment,
|
||||
bool *zero, bool *commit);
|
||||
bool chunk_dalloc_mmap(void *chunk, size_t size);
|
||||
bool pages_purge(void *addr, size_t length);
|
||||
|
||||
void *chunk_alloc_mmap(size_t size, size_t alignment, bool *zero);
|
||||
bool chunk_dealloc_mmap(void *chunk, size_t size);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
|
||||
@@ -40,7 +40,9 @@ struct ckh_s {
|
||||
#endif
|
||||
|
||||
/* Used for pseudo-random number generation. */
|
||||
uint64_t prng_state;
|
||||
#define CKH_A 1103515241
|
||||
#define CKH_C 12347
|
||||
uint32_t prng_state;
|
||||
|
||||
/* Total number of items. */
|
||||
size_t count;
|
||||
@@ -64,15 +66,15 @@ struct ckh_s {
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
bool ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
|
||||
bool ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
|
||||
ckh_keycomp_t *keycomp);
|
||||
void ckh_delete(tsd_t *tsd, ckh_t *ckh);
|
||||
void ckh_delete(ckh_t *ckh);
|
||||
size_t ckh_count(ckh_t *ckh);
|
||||
bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data);
|
||||
bool ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data);
|
||||
bool ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
|
||||
bool ckh_insert(ckh_t *ckh, const void *key, const void *data);
|
||||
bool ckh_remove(ckh_t *ckh, const void *searchkey, void **key,
|
||||
void **data);
|
||||
bool ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data);
|
||||
bool ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data);
|
||||
void ckh_string_hash(const void *key, size_t r_hash[2]);
|
||||
bool ckh_string_keycomp(const void *k1, const void *k2);
|
||||
void ckh_pointer_hash(const void *key, size_t r_hash[2]);
|
||||
|
||||
@@ -21,27 +21,21 @@ struct ctl_named_node_s {
|
||||
/* If (nchildren == 0), this is a terminal node. */
|
||||
unsigned nchildren;
|
||||
const ctl_node_t *children;
|
||||
int (*ctl)(tsd_t *, const size_t *, size_t, void *,
|
||||
size_t *, void *, size_t);
|
||||
int (*ctl)(const size_t *, size_t, void *, size_t *,
|
||||
void *, size_t);
|
||||
};
|
||||
|
||||
struct ctl_indexed_node_s {
|
||||
struct ctl_node_s node;
|
||||
const ctl_named_node_t *(*index)(tsdn_t *, const size_t *, size_t,
|
||||
size_t);
|
||||
const ctl_named_node_t *(*index)(const size_t *, size_t, size_t);
|
||||
};
|
||||
|
||||
struct ctl_arena_stats_s {
|
||||
bool initialized;
|
||||
unsigned nthreads;
|
||||
const char *dss;
|
||||
ssize_t lg_dirty_mult;
|
||||
ssize_t decay_time;
|
||||
size_t pactive;
|
||||
size_t pdirty;
|
||||
|
||||
/* The remainder are only populated if config_stats is true. */
|
||||
|
||||
arena_stats_t astats;
|
||||
|
||||
/* Aggregate stats for small size classes, based on bin stats. */
|
||||
@@ -52,16 +46,22 @@ struct ctl_arena_stats_s {
|
||||
|
||||
malloc_bin_stats_t bstats[NBINS];
|
||||
malloc_large_stats_t *lstats; /* nlclasses elements. */
|
||||
malloc_huge_stats_t *hstats; /* nhclasses elements. */
|
||||
};
|
||||
|
||||
struct ctl_stats_s {
|
||||
size_t allocated;
|
||||
size_t active;
|
||||
size_t metadata;
|
||||
size_t resident;
|
||||
size_t mapped;
|
||||
size_t retained;
|
||||
struct {
|
||||
size_t current; /* stats_chunks.curchunks */
|
||||
uint64_t total; /* stats_chunks.nchunks */
|
||||
size_t high; /* stats_chunks.highchunks */
|
||||
} chunks;
|
||||
struct {
|
||||
size_t allocated; /* huge_allocated */
|
||||
uint64_t nmalloc; /* huge_nmalloc */
|
||||
uint64_t ndalloc; /* huge_ndalloc */
|
||||
} huge;
|
||||
unsigned narenas;
|
||||
ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */
|
||||
};
|
||||
@@ -70,17 +70,16 @@ struct ctl_stats_s {
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
int ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
|
||||
void *newp, size_t newlen);
|
||||
int ctl_nametomib(tsdn_t *tsdn, const char *name, size_t *mibp,
|
||||
size_t *miblenp);
|
||||
int ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp,
|
||||
size_t newlen);
|
||||
int ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp);
|
||||
|
||||
int ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
|
||||
size_t *oldlenp, void *newp, size_t newlen);
|
||||
int ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
|
||||
void *newp, size_t newlen);
|
||||
bool ctl_boot(void);
|
||||
void ctl_prefork(tsdn_t *tsdn);
|
||||
void ctl_postfork_parent(tsdn_t *tsdn);
|
||||
void ctl_postfork_child(tsdn_t *tsdn);
|
||||
void ctl_prefork(void);
|
||||
void ctl_postfork_parent(void);
|
||||
void ctl_postfork_child(void);
|
||||
|
||||
#define xmallctl(name, oldp, oldlenp, newp, newlen) do { \
|
||||
if (je_mallctl(name, oldp, oldlenp, newp, newlen) \
|
||||
|
||||
@@ -7,67 +7,25 @@ typedef struct extent_node_s extent_node_t;
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
|
||||
/* Tree of extents. Use accessor functions for en_* fields. */
|
||||
/* Tree of extents. */
|
||||
struct extent_node_s {
|
||||
/* Arena from which this extent came, if any. */
|
||||
arena_t *en_arena;
|
||||
|
||||
/* Pointer to the extent that this tree node is responsible for. */
|
||||
void *en_addr;
|
||||
|
||||
/* Total region size. */
|
||||
size_t en_size;
|
||||
|
||||
/*
|
||||
* Serial number (potentially non-unique).
|
||||
*
|
||||
* In principle serial numbers can wrap around on 32-bit systems if
|
||||
* JEMALLOC_MUNMAP is defined, but as long as comparison functions fall
|
||||
* back on address comparison for equal serial numbers, stable (if
|
||||
* imperfect) ordering is maintained.
|
||||
*
|
||||
* Serial numbers may not be unique even in the absence of wrap-around,
|
||||
* e.g. when splitting an extent and assigning the same serial number to
|
||||
* both resulting adjacent extents.
|
||||
*/
|
||||
size_t en_sn;
|
||||
|
||||
/*
|
||||
* The zeroed flag is used by chunk recycling code to track whether
|
||||
* memory is zero-filled.
|
||||
*/
|
||||
bool en_zeroed;
|
||||
|
||||
/*
|
||||
* True if physical memory is committed to the extent, whether
|
||||
* explicitly or implicitly as on a system that overcommits and
|
||||
* satisfies physical memory needs on demand via soft page faults.
|
||||
*/
|
||||
bool en_committed;
|
||||
|
||||
/*
|
||||
* The achunk flag is used to validate that huge allocation lookups
|
||||
* don't return arena chunks.
|
||||
*/
|
||||
bool en_achunk;
|
||||
|
||||
/* Profile counters, used for huge objects. */
|
||||
prof_tctx_t *en_prof_tctx;
|
||||
|
||||
/* Linkage for arena's runs_dirty and chunks_cache rings. */
|
||||
arena_runs_dirty_link_t rd;
|
||||
qr(extent_node_t) cc_link;
|
||||
|
||||
union {
|
||||
/* Linkage for the size/sn/address-ordered tree. */
|
||||
rb_node(extent_node_t) szsnad_link;
|
||||
|
||||
/* Linkage for arena's achunks, huge, and node_cache lists. */
|
||||
ql_elm(extent_node_t) ql_link;
|
||||
};
|
||||
/* Linkage for the size/address-ordered tree. */
|
||||
rb_node(extent_node_t) link_szad;
|
||||
|
||||
/* Linkage for the address-ordered tree. */
|
||||
rb_node(extent_node_t) ad_link;
|
||||
rb_node(extent_node_t) link_ad;
|
||||
|
||||
/* Profile counters, used for huge objects. */
|
||||
prof_ctx_t *prof_ctx;
|
||||
|
||||
/* Pointer to the extent that this tree node is responsible for. */
|
||||
void *addr;
|
||||
|
||||
/* Total region size. */
|
||||
size_t size;
|
||||
|
||||
/* True if zero-filled; used by chunk recycling code. */
|
||||
bool zeroed;
|
||||
};
|
||||
typedef rb_tree(extent_node_t) extent_tree_t;
|
||||
|
||||
@@ -75,7 +33,7 @@ typedef rb_tree(extent_node_t) extent_tree_t;
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
rb_proto(, extent_tree_szsnad_, extent_tree_t, extent_node_t)
|
||||
rb_proto(, extent_tree_szad_, extent_tree_t, extent_node_t)
|
||||
|
||||
rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t)
|
||||
|
||||
@@ -83,188 +41,6 @@ rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t)
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
arena_t *extent_node_arena_get(const extent_node_t *node);
|
||||
void *extent_node_addr_get(const extent_node_t *node);
|
||||
size_t extent_node_size_get(const extent_node_t *node);
|
||||
size_t extent_node_sn_get(const extent_node_t *node);
|
||||
bool extent_node_zeroed_get(const extent_node_t *node);
|
||||
bool extent_node_committed_get(const extent_node_t *node);
|
||||
bool extent_node_achunk_get(const extent_node_t *node);
|
||||
prof_tctx_t *extent_node_prof_tctx_get(const extent_node_t *node);
|
||||
void extent_node_arena_set(extent_node_t *node, arena_t *arena);
|
||||
void extent_node_addr_set(extent_node_t *node, void *addr);
|
||||
void extent_node_size_set(extent_node_t *node, size_t size);
|
||||
void extent_node_sn_set(extent_node_t *node, size_t sn);
|
||||
void extent_node_zeroed_set(extent_node_t *node, bool zeroed);
|
||||
void extent_node_committed_set(extent_node_t *node, bool committed);
|
||||
void extent_node_achunk_set(extent_node_t *node, bool achunk);
|
||||
void extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx);
|
||||
void extent_node_init(extent_node_t *node, arena_t *arena, void *addr,
|
||||
size_t size, size_t sn, bool zeroed, bool committed);
|
||||
void extent_node_dirty_linkage_init(extent_node_t *node);
|
||||
void extent_node_dirty_insert(extent_node_t *node,
|
||||
arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty);
|
||||
void extent_node_dirty_remove(extent_node_t *node);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_))
|
||||
JEMALLOC_INLINE arena_t *
|
||||
extent_node_arena_get(const extent_node_t *node)
|
||||
{
|
||||
|
||||
return (node->en_arena);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void *
|
||||
extent_node_addr_get(const extent_node_t *node)
|
||||
{
|
||||
|
||||
return (node->en_addr);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE size_t
|
||||
extent_node_size_get(const extent_node_t *node)
|
||||
{
|
||||
|
||||
return (node->en_size);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE size_t
|
||||
extent_node_sn_get(const extent_node_t *node)
|
||||
{
|
||||
|
||||
return (node->en_sn);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
extent_node_zeroed_get(const extent_node_t *node)
|
||||
{
|
||||
|
||||
return (node->en_zeroed);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
extent_node_committed_get(const extent_node_t *node)
|
||||
{
|
||||
|
||||
assert(!node->en_achunk);
|
||||
return (node->en_committed);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
extent_node_achunk_get(const extent_node_t *node)
|
||||
{
|
||||
|
||||
return (node->en_achunk);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE prof_tctx_t *
|
||||
extent_node_prof_tctx_get(const extent_node_t *node)
|
||||
{
|
||||
|
||||
return (node->en_prof_tctx);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
extent_node_arena_set(extent_node_t *node, arena_t *arena)
|
||||
{
|
||||
|
||||
node->en_arena = arena;
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
extent_node_addr_set(extent_node_t *node, void *addr)
|
||||
{
|
||||
|
||||
node->en_addr = addr;
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
extent_node_size_set(extent_node_t *node, size_t size)
|
||||
{
|
||||
|
||||
node->en_size = size;
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
extent_node_sn_set(extent_node_t *node, size_t sn)
|
||||
{
|
||||
|
||||
node->en_sn = sn;
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
extent_node_zeroed_set(extent_node_t *node, bool zeroed)
|
||||
{
|
||||
|
||||
node->en_zeroed = zeroed;
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
extent_node_committed_set(extent_node_t *node, bool committed)
|
||||
{
|
||||
|
||||
node->en_committed = committed;
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
extent_node_achunk_set(extent_node_t *node, bool achunk)
|
||||
{
|
||||
|
||||
node->en_achunk = achunk;
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx)
|
||||
{
|
||||
|
||||
node->en_prof_tctx = tctx;
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size,
|
||||
size_t sn, bool zeroed, bool committed)
|
||||
{
|
||||
|
||||
extent_node_arena_set(node, arena);
|
||||
extent_node_addr_set(node, addr);
|
||||
extent_node_size_set(node, size);
|
||||
extent_node_sn_set(node, sn);
|
||||
extent_node_zeroed_set(node, zeroed);
|
||||
extent_node_committed_set(node, committed);
|
||||
extent_node_achunk_set(node, false);
|
||||
if (config_prof)
|
||||
extent_node_prof_tctx_set(node, NULL);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
extent_node_dirty_linkage_init(extent_node_t *node)
|
||||
{
|
||||
|
||||
qr_new(&node->rd, rd_link);
|
||||
qr_new(node, cc_link);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
extent_node_dirty_insert(extent_node_t *node,
|
||||
arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty)
|
||||
{
|
||||
|
||||
qr_meld(runs_dirty, &node->rd, rd_link);
|
||||
qr_meld(chunks_dirty, node, cc_link);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
extent_node_dirty_remove(extent_node_t *node)
|
||||
{
|
||||
|
||||
qr_remove(&node->rd, rd_link);
|
||||
qr_remove(node, cc_link);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/*
|
||||
* The following hash function is based on MurmurHash3, placed into the public
|
||||
* domain by Austin Appleby. See https://github.com/aappleby/smhasher for
|
||||
* domain by Austin Appleby. See http://code.google.com/p/smhasher/ for
|
||||
* details.
|
||||
*/
|
||||
/******************************************************************************/
|
||||
@@ -35,28 +35,19 @@ JEMALLOC_INLINE uint32_t
|
||||
hash_rotl_32(uint32_t x, int8_t r)
|
||||
{
|
||||
|
||||
return ((x << r) | (x >> (32 - r)));
|
||||
return (x << r) | (x >> (32 - r));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint64_t
|
||||
hash_rotl_64(uint64_t x, int8_t r)
|
||||
{
|
||||
|
||||
return ((x << r) | (x >> (64 - r)));
|
||||
return (x << r) | (x >> (64 - r));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint32_t
|
||||
hash_get_block_32(const uint32_t *p, int i)
|
||||
{
|
||||
|
||||
/* Handle unaligned read. */
|
||||
if (unlikely((uintptr_t)p & (sizeof(uint32_t)-1)) != 0) {
|
||||
uint32_t ret;
|
||||
|
||||
memcpy(&ret, (uint8_t *)(p + i), sizeof(uint32_t));
|
||||
return (ret);
|
||||
}
|
||||
|
||||
return (p[i]);
|
||||
}
|
||||
|
||||
@@ -64,14 +55,6 @@ JEMALLOC_INLINE uint64_t
|
||||
hash_get_block_64(const uint64_t *p, int i)
|
||||
{
|
||||
|
||||
/* Handle unaligned read. */
|
||||
if (unlikely((uintptr_t)p & (sizeof(uint64_t)-1)) != 0) {
|
||||
uint64_t ret;
|
||||
|
||||
memcpy(&ret, (uint8_t *)(p + i), sizeof(uint64_t));
|
||||
return (ret);
|
||||
}
|
||||
|
||||
return (p[i]);
|
||||
}
|
||||
|
||||
@@ -93,9 +76,9 @@ hash_fmix_64(uint64_t k)
|
||||
{
|
||||
|
||||
k ^= k >> 33;
|
||||
k *= KQU(0xff51afd7ed558ccd);
|
||||
k *= QU(0xff51afd7ed558ccdLLU);
|
||||
k ^= k >> 33;
|
||||
k *= KQU(0xc4ceb9fe1a85ec53);
|
||||
k *= QU(0xc4ceb9fe1a85ec53LLU);
|
||||
k ^= k >> 33;
|
||||
|
||||
return (k);
|
||||
@@ -264,8 +247,8 @@ hash_x64_128(const void *key, const int len, const uint32_t seed,
|
||||
uint64_t h1 = seed;
|
||||
uint64_t h2 = seed;
|
||||
|
||||
const uint64_t c1 = KQU(0x87c37b91114253d5);
|
||||
const uint64_t c2 = KQU(0x4cf5ad432745937f);
|
||||
const uint64_t c1 = QU(0x87c37b91114253d5LLU);
|
||||
const uint64_t c2 = QU(0x4cf5ad432745937fLLU);
|
||||
|
||||
/* body */
|
||||
{
|
||||
@@ -337,18 +320,13 @@ hash_x64_128(const void *key, const int len, const uint32_t seed,
|
||||
JEMALLOC_INLINE void
|
||||
hash(const void *key, size_t len, const uint32_t seed, size_t r_hash[2])
|
||||
{
|
||||
|
||||
assert(len <= INT_MAX); /* Unfortunate implementation limitation. */
|
||||
|
||||
#if (LG_SIZEOF_PTR == 3 && !defined(JEMALLOC_BIG_ENDIAN))
|
||||
hash_x64_128(key, (int)len, seed, (uint64_t *)r_hash);
|
||||
hash_x64_128(key, len, seed, (uint64_t *)r_hash);
|
||||
#else
|
||||
{
|
||||
uint64_t hashes[2];
|
||||
hash_x86_128(key, (int)len, seed, hashes);
|
||||
r_hash[0] = (size_t)hashes[0];
|
||||
r_hash[1] = (size_t)hashes[1];
|
||||
}
|
||||
uint64_t hashes[2];
|
||||
hash_x86_128(key, len, seed, hashes);
|
||||
r_hash[0] = (size_t)hashes[0];
|
||||
r_hash[1] = (size_t)hashes[1];
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -9,23 +9,34 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
void *huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
|
||||
void *huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
||||
size_t alignment, bool zero);
|
||||
bool huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
||||
size_t usize_min, size_t usize_max, bool zero);
|
||||
void *huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
|
||||
size_t usize, size_t alignment, bool zero, tcache_t *tcache);
|
||||
/* Huge allocation statistics. */
|
||||
extern uint64_t huge_nmalloc;
|
||||
extern uint64_t huge_ndalloc;
|
||||
extern size_t huge_allocated;
|
||||
|
||||
/* Protects chunk-related data structures. */
|
||||
extern malloc_mutex_t huge_mtx;
|
||||
|
||||
void *huge_malloc(size_t size, bool zero, dss_prec_t dss_prec);
|
||||
void *huge_palloc(size_t size, size_t alignment, bool zero,
|
||||
dss_prec_t dss_prec);
|
||||
bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
|
||||
size_t extra);
|
||||
void *huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
|
||||
size_t alignment, bool zero, bool try_tcache_dalloc, dss_prec_t dss_prec);
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef void (huge_dalloc_junk_t)(void *, size_t);
|
||||
extern huge_dalloc_junk_t *huge_dalloc_junk;
|
||||
#endif
|
||||
void huge_dalloc(tsdn_t *tsdn, void *ptr);
|
||||
arena_t *huge_aalloc(const void *ptr);
|
||||
size_t huge_salloc(tsdn_t *tsdn, const void *ptr);
|
||||
prof_tctx_t *huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr);
|
||||
void huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx);
|
||||
void huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr);
|
||||
void huge_dalloc(void *ptr, bool unmap);
|
||||
size_t huge_salloc(const void *ptr);
|
||||
dss_prec_t huge_dss_prec_get(arena_t *arena);
|
||||
prof_ctx_t *huge_prof_ctx_get(const void *ptr);
|
||||
void huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
|
||||
bool huge_boot(void);
|
||||
void huge_prefork(void);
|
||||
void huge_postfork_parent(void);
|
||||
void huge_postfork_child(void);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,75 +0,0 @@
|
||||
#ifndef JEMALLOC_INTERNAL_DECLS_H
|
||||
#define JEMALLOC_INTERNAL_DECLS_H
|
||||
|
||||
#include <math.h>
|
||||
#ifdef _WIN32
|
||||
# include <windows.h>
|
||||
# include "msvc_compat/windows_extra.h"
|
||||
|
||||
#else
|
||||
# include <sys/param.h>
|
||||
# include <sys/mman.h>
|
||||
# if !defined(__pnacl__) && !defined(__native_client__)
|
||||
# include <sys/syscall.h>
|
||||
# if !defined(SYS_write) && defined(__NR_write)
|
||||
# define SYS_write __NR_write
|
||||
# endif
|
||||
# include <sys/uio.h>
|
||||
# endif
|
||||
# include <pthread.h>
|
||||
# ifdef JEMALLOC_OS_UNFAIR_LOCK
|
||||
# include <os/lock.h>
|
||||
# endif
|
||||
# ifdef JEMALLOC_GLIBC_MALLOC_HOOK
|
||||
# include <sched.h>
|
||||
# endif
|
||||
# include <errno.h>
|
||||
# include <sys/time.h>
|
||||
# include <time.h>
|
||||
# ifdef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME
|
||||
# include <mach/mach_time.h>
|
||||
# endif
|
||||
#endif
|
||||
#include <sys/types.h>
|
||||
|
||||
#include <limits.h>
|
||||
#ifndef SIZE_T_MAX
|
||||
# define SIZE_T_MAX SIZE_MAX
|
||||
#endif
|
||||
#include <stdarg.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#ifndef offsetof
|
||||
# define offsetof(type, member) ((size_t)&(((type *)NULL)->member))
|
||||
#endif
|
||||
#include <string.h>
|
||||
#include <strings.h>
|
||||
#include <ctype.h>
|
||||
#ifdef _MSC_VER
|
||||
# include <io.h>
|
||||
typedef intptr_t ssize_t;
|
||||
# define PATH_MAX 1024
|
||||
# define STDERR_FILENO 2
|
||||
# define __func__ __FUNCTION__
|
||||
# ifdef JEMALLOC_HAS_RESTRICT
|
||||
# define restrict __restrict
|
||||
# endif
|
||||
/* Disable warnings about deprecated system functions. */
|
||||
# pragma warning(disable: 4996)
|
||||
#if _MSC_VER < 1800
|
||||
static int
|
||||
isblank(int c)
|
||||
{
|
||||
|
||||
return (c == '\t' || c == ' ');
|
||||
}
|
||||
#endif
|
||||
#else
|
||||
# include <unistd.h>
|
||||
#endif
|
||||
#include <fcntl.h>
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_H */
|
||||
@@ -22,9 +22,6 @@
|
||||
*/
|
||||
#undef CPU_SPINWAIT
|
||||
|
||||
/* Defined if C11 atomics are available. */
|
||||
#undef JEMALLOC_C11ATOMICS
|
||||
|
||||
/* Defined if the equivalent of FreeBSD's atomic(9) functions are available. */
|
||||
#undef JEMALLOC_ATOMIC9
|
||||
|
||||
@@ -38,7 +35,7 @@
|
||||
* Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and
|
||||
* __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite
|
||||
* __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the
|
||||
* functions are defined in libgcc instead of being inlines).
|
||||
* functions are defined in libgcc instead of being inlines)
|
||||
*/
|
||||
#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4
|
||||
|
||||
@@ -46,57 +43,16 @@
|
||||
* Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and
|
||||
* __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite
|
||||
* __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the
|
||||
* functions are defined in libgcc instead of being inlines).
|
||||
* functions are defined in libgcc instead of being inlines)
|
||||
*/
|
||||
#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8
|
||||
|
||||
/*
|
||||
* Defined if __builtin_clz() and __builtin_clzl() are available.
|
||||
*/
|
||||
#undef JEMALLOC_HAVE_BUILTIN_CLZ
|
||||
|
||||
/*
|
||||
* Defined if os_unfair_lock_*() functions are available, as provided by Darwin.
|
||||
*/
|
||||
#undef JEMALLOC_OS_UNFAIR_LOCK
|
||||
|
||||
/*
|
||||
* Defined if OSSpin*() functions are available, as provided by Darwin, and
|
||||
* documented in the spinlock(3) manual page.
|
||||
*/
|
||||
#undef JEMALLOC_OSSPIN
|
||||
|
||||
/* Defined if syscall(2) is usable. */
|
||||
#undef JEMALLOC_USE_SYSCALL
|
||||
|
||||
/*
|
||||
* Defined if secure_getenv(3) is available.
|
||||
*/
|
||||
#undef JEMALLOC_HAVE_SECURE_GETENV
|
||||
|
||||
/*
|
||||
* Defined if issetugid(2) is available.
|
||||
*/
|
||||
#undef JEMALLOC_HAVE_ISSETUGID
|
||||
|
||||
/* Defined if pthread_atfork(3) is available. */
|
||||
#undef JEMALLOC_HAVE_PTHREAD_ATFORK
|
||||
|
||||
/*
|
||||
* Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
|
||||
*/
|
||||
#undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
|
||||
|
||||
/*
|
||||
* Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.
|
||||
*/
|
||||
#undef JEMALLOC_HAVE_CLOCK_MONOTONIC
|
||||
|
||||
/*
|
||||
* Defined if mach_absolute_time() is available.
|
||||
*/
|
||||
#undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME
|
||||
|
||||
/*
|
||||
* Defined if _malloc_thread_cleanup() exists. At least in the case of
|
||||
* FreeBSD, pthread_key_create() allocates, which if used during malloc
|
||||
@@ -120,6 +76,9 @@
|
||||
*/
|
||||
#undef JEMALLOC_MUTEX_INIT_CB
|
||||
|
||||
/* Defined if sbrk() is supported. */
|
||||
#undef JEMALLOC_HAVE_SBRK
|
||||
|
||||
/* Non-empty if the tls_model attribute is supported. */
|
||||
#undef JEMALLOC_TLS_MODEL
|
||||
|
||||
@@ -178,26 +137,8 @@
|
||||
/* Support lazy locking (avoid locking unless a second thread is launched). */
|
||||
#undef JEMALLOC_LAZY_LOCK
|
||||
|
||||
/* Minimum size class to support is 2^LG_TINY_MIN bytes. */
|
||||
#undef LG_TINY_MIN
|
||||
|
||||
/*
|
||||
* Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size
|
||||
* classes).
|
||||
*/
|
||||
#undef LG_QUANTUM
|
||||
|
||||
/* One page is 2^LG_PAGE bytes. */
|
||||
#undef LG_PAGE
|
||||
|
||||
/*
|
||||
* If defined, adjacent virtual memory mappings with identical attributes
|
||||
* automatically coalesce, and they fragment when changes are made to subranges.
|
||||
* This is the normal order of things for mmap()/munmap(), but on Windows
|
||||
* VirtualAlloc()/VirtualFree() operations must be precisely matched, i.e.
|
||||
* mappings do *not* coalesce/fragment.
|
||||
*/
|
||||
#undef JEMALLOC_MAPS_COALESCE
|
||||
/* One page is 2^STATIC_PAGE_SHIFT bytes. */
|
||||
#undef STATIC_PAGE_SHIFT
|
||||
|
||||
/*
|
||||
* If defined, use munmap() to unmap freed chunks, rather than storing them for
|
||||
@@ -206,72 +147,44 @@
|
||||
*/
|
||||
#undef JEMALLOC_MUNMAP
|
||||
|
||||
/*
|
||||
* If defined, use mremap(...MREMAP_FIXED...) for huge realloc(). This is
|
||||
* disabled by default because it is Linux-specific and it will cause virtual
|
||||
* memory map holes, much like munmap(2) does.
|
||||
*/
|
||||
#undef JEMALLOC_MREMAP
|
||||
|
||||
/* TLS is used to map arenas and magazine caches to threads. */
|
||||
#undef JEMALLOC_TLS
|
||||
|
||||
/*
|
||||
* Used to mark unreachable code to quiet "end of non-void" compiler warnings.
|
||||
* Don't use this directly; instead use unreachable() from util.h
|
||||
*/
|
||||
#undef JEMALLOC_INTERNAL_UNREACHABLE
|
||||
|
||||
/*
|
||||
* ffs*() functions to use for bitmapping. Don't use these directly; instead,
|
||||
* use ffs_*() from util.h.
|
||||
*/
|
||||
#undef JEMALLOC_INTERNAL_FFSLL
|
||||
#undef JEMALLOC_INTERNAL_FFSL
|
||||
#undef JEMALLOC_INTERNAL_FFS
|
||||
|
||||
/*
|
||||
* JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside
|
||||
* within jemalloc-owned chunks before dereferencing them.
|
||||
*/
|
||||
#undef JEMALLOC_IVSALLOC
|
||||
|
||||
/*
|
||||
* If defined, explicitly attempt to more uniformly distribute large allocation
|
||||
* pointer alignments across all cache indices.
|
||||
*/
|
||||
#undef JEMALLOC_CACHE_OBLIVIOUS
|
||||
|
||||
/*
|
||||
* Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
|
||||
*/
|
||||
#undef JEMALLOC_ZONE
|
||||
#undef JEMALLOC_ZONE_VERSION
|
||||
|
||||
/*
|
||||
* Methods for determining whether the OS overcommits.
|
||||
* JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY: Linux's
|
||||
* /proc/sys/vm.overcommit_memory file.
|
||||
* JEMALLOC_SYSCTL_VM_OVERCOMMIT: FreeBSD's vm.overcommit sysctl.
|
||||
*/
|
||||
#undef JEMALLOC_SYSCTL_VM_OVERCOMMIT
|
||||
#undef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
|
||||
|
||||
/* Defined if madvise(2) is available. */
|
||||
#undef JEMALLOC_HAVE_MADVISE
|
||||
|
||||
/*
|
||||
* Methods for purging unused pages differ between operating systems.
|
||||
*
|
||||
* madvise(..., MADV_FREE) : This marks pages as being unused, such that they
|
||||
* will be discarded rather than swapped out.
|
||||
* madvise(..., MADV_DONTNEED) : This immediately discards pages, such that
|
||||
* new pages will be demand-zeroed if the
|
||||
* address region is later touched.
|
||||
* madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages,
|
||||
* such that new pages will be demand-zeroed if
|
||||
* the address region is later touched.
|
||||
* madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being
|
||||
* unused, such that they will be discarded rather
|
||||
* than swapped out.
|
||||
*/
|
||||
#undef JEMALLOC_PURGE_MADVISE_FREE
|
||||
#undef JEMALLOC_PURGE_MADVISE_DONTNEED
|
||||
#undef JEMALLOC_PURGE_MADVISE_FREE
|
||||
|
||||
/*
|
||||
* Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE
|
||||
* arguments to madvise(2).
|
||||
* Define if operating system has alloca.h header.
|
||||
*/
|
||||
#undef JEMALLOC_THP
|
||||
|
||||
/* Define if operating system has alloca.h header. */
|
||||
#undef JEMALLOC_HAS_ALLOCA_H
|
||||
|
||||
/* C99 restrict keyword supported. */
|
||||
@@ -286,28 +199,7 @@
|
||||
/* sizeof(long) == 2^LG_SIZEOF_LONG. */
|
||||
#undef LG_SIZEOF_LONG
|
||||
|
||||
/* sizeof(long long) == 2^LG_SIZEOF_LONG_LONG. */
|
||||
#undef LG_SIZEOF_LONG_LONG
|
||||
|
||||
/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */
|
||||
#undef LG_SIZEOF_INTMAX_T
|
||||
|
||||
/* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook). */
|
||||
#undef JEMALLOC_GLIBC_MALLOC_HOOK
|
||||
|
||||
/* glibc memalign hook. */
|
||||
#undef JEMALLOC_GLIBC_MEMALIGN_HOOK
|
||||
|
||||
/* Adaptive mutex support in pthreads. */
|
||||
#undef JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP
|
||||
|
||||
/*
|
||||
* If defined, jemalloc symbols are not exported (doesn't work when
|
||||
* JEMALLOC_PREFIX is not defined).
|
||||
*/
|
||||
#undef JEMALLOC_EXPORT
|
||||
|
||||
/* config.malloc_conf options string. */
|
||||
#undef JEMALLOC_CONFIG_MALLOC_CONF
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_DEFS_H_ */
|
||||
|
||||
@@ -39,15 +39,9 @@
|
||||
#endif
|
||||
|
||||
#define ZU(z) ((size_t)z)
|
||||
#define ZI(z) ((ssize_t)z)
|
||||
#define QU(q) ((uint64_t)q)
|
||||
#define QI(q) ((int64_t)q)
|
||||
|
||||
#define KZU(z) ZU(z##ULL)
|
||||
#define KZI(z) ZI(z##LL)
|
||||
#define KQU(q) QU(q##ULL)
|
||||
#define KQI(q) QI(q##LL)
|
||||
|
||||
#ifndef __DECONST
|
||||
# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
|
||||
#endif
|
||||
|
||||
@@ -42,7 +42,7 @@ mb_write(void)
|
||||
: /* Inputs. */
|
||||
: "memory" /* Clobbers. */
|
||||
);
|
||||
# else
|
||||
#else
|
||||
/*
|
||||
* This is hopefully enough to keep the compiler from reordering
|
||||
* instructions around this one.
|
||||
@@ -52,7 +52,7 @@ mb_write(void)
|
||||
: /* Inputs. */
|
||||
: "memory" /* Clobbers. */
|
||||
);
|
||||
# endif
|
||||
#endif
|
||||
}
|
||||
#elif (defined(__amd64__) || defined(__x86_64__))
|
||||
JEMALLOC_INLINE void
|
||||
@@ -104,9 +104,9 @@ mb_write(void)
|
||||
{
|
||||
malloc_mutex_t mtx;
|
||||
|
||||
malloc_mutex_init(&mtx, "mb", WITNESS_RANK_OMIT);
|
||||
malloc_mutex_lock(TSDN_NULL, &mtx);
|
||||
malloc_mutex_unlock(TSDN_NULL, &mtx);
|
||||
malloc_mutex_init(&mtx);
|
||||
malloc_mutex_lock(&mtx);
|
||||
malloc_mutex_unlock(&mtx);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@@ -5,25 +5,18 @@ typedef struct malloc_mutex_s malloc_mutex_t;
|
||||
|
||||
#ifdef _WIN32
|
||||
# define MALLOC_MUTEX_INITIALIZER
|
||||
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
|
||||
# define MALLOC_MUTEX_INITIALIZER \
|
||||
{OS_UNFAIR_LOCK_INIT, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
|
||||
#elif (defined(JEMALLOC_OSSPIN))
|
||||
# define MALLOC_MUTEX_INITIALIZER {0, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
|
||||
# define MALLOC_MUTEX_INITIALIZER {0}
|
||||
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
|
||||
# define MALLOC_MUTEX_INITIALIZER \
|
||||
{PTHREAD_MUTEX_INITIALIZER, NULL, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
|
||||
# define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER, NULL}
|
||||
#else
|
||||
# if (defined(JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP) && \
|
||||
# if (defined(PTHREAD_MUTEX_ADAPTIVE_NP) && \
|
||||
defined(PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP))
|
||||
# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_ADAPTIVE_NP
|
||||
# define MALLOC_MUTEX_INITIALIZER \
|
||||
{PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP, \
|
||||
WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
|
||||
# define MALLOC_MUTEX_INITIALIZER {PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP}
|
||||
# else
|
||||
# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
|
||||
# define MALLOC_MUTEX_INITIALIZER \
|
||||
{PTHREAD_MUTEX_INITIALIZER, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
|
||||
# define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER}
|
||||
# endif
|
||||
#endif
|
||||
|
||||
@@ -33,13 +26,7 @@ typedef struct malloc_mutex_s malloc_mutex_t;
|
||||
|
||||
struct malloc_mutex_s {
|
||||
#ifdef _WIN32
|
||||
# if _WIN32_WINNT >= 0x0600
|
||||
SRWLOCK lock;
|
||||
# else
|
||||
CRITICAL_SECTION lock;
|
||||
# endif
|
||||
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
|
||||
os_unfair_lock lock;
|
||||
#elif (defined(JEMALLOC_OSSPIN))
|
||||
OSSpinLock lock;
|
||||
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
|
||||
@@ -48,7 +35,6 @@ struct malloc_mutex_s {
|
||||
#else
|
||||
pthread_mutex_t lock;
|
||||
#endif
|
||||
witness_t witness;
|
||||
};
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
@@ -62,62 +48,44 @@ extern bool isthreaded;
|
||||
# define isthreaded true
|
||||
#endif
|
||||
|
||||
bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
|
||||
witness_rank_t rank);
|
||||
void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
|
||||
void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
|
||||
void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
|
||||
bool malloc_mutex_boot(void);
|
||||
bool malloc_mutex_init(malloc_mutex_t *mutex);
|
||||
void malloc_mutex_prefork(malloc_mutex_t *mutex);
|
||||
void malloc_mutex_postfork_parent(malloc_mutex_t *mutex);
|
||||
void malloc_mutex_postfork_child(malloc_mutex_t *mutex);
|
||||
bool mutex_boot(void);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
void malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex);
|
||||
void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex);
|
||||
void malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex);
|
||||
void malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex);
|
||||
void malloc_mutex_lock(malloc_mutex_t *mutex);
|
||||
void malloc_mutex_unlock(malloc_mutex_t *mutex);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
|
||||
JEMALLOC_INLINE void
|
||||
malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
malloc_mutex_lock(malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
if (isthreaded) {
|
||||
witness_assert_not_owner(tsdn, &mutex->witness);
|
||||
#ifdef _WIN32
|
||||
# if _WIN32_WINNT >= 0x0600
|
||||
AcquireSRWLockExclusive(&mutex->lock);
|
||||
# else
|
||||
EnterCriticalSection(&mutex->lock);
|
||||
# endif
|
||||
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
|
||||
os_unfair_lock_lock(&mutex->lock);
|
||||
#elif (defined(JEMALLOC_OSSPIN))
|
||||
OSSpinLockLock(&mutex->lock);
|
||||
#else
|
||||
pthread_mutex_lock(&mutex->lock);
|
||||
#endif
|
||||
witness_lock(tsdn, &mutex->witness);
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
malloc_mutex_unlock(malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
if (isthreaded) {
|
||||
witness_unlock(tsdn, &mutex->witness);
|
||||
#ifdef _WIN32
|
||||
# if _WIN32_WINNT >= 0x0600
|
||||
ReleaseSRWLockExclusive(&mutex->lock);
|
||||
# else
|
||||
LeaveCriticalSection(&mutex->lock);
|
||||
# endif
|
||||
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
|
||||
os_unfair_lock_unlock(&mutex->lock);
|
||||
#elif (defined(JEMALLOC_OSSPIN))
|
||||
OSSpinLockUnlock(&mutex->lock);
|
||||
#else
|
||||
@@ -125,22 +93,6 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
if (isthreaded)
|
||||
witness_assert_owner(tsdn, &mutex->witness);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
if (isthreaded)
|
||||
witness_assert_not_owner(tsdn, &mutex->witness);
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
typedef struct nstime_s nstime_t;
|
||||
|
||||
/* Maximum supported number of seconds (~584 years). */
|
||||
#define NSTIME_SEC_MAX KQU(18446744072)
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
|
||||
struct nstime_s {
|
||||
uint64_t ns;
|
||||
};
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
void nstime_init(nstime_t *time, uint64_t ns);
|
||||
void nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec);
|
||||
uint64_t nstime_ns(const nstime_t *time);
|
||||
uint64_t nstime_sec(const nstime_t *time);
|
||||
uint64_t nstime_nsec(const nstime_t *time);
|
||||
void nstime_copy(nstime_t *time, const nstime_t *source);
|
||||
int nstime_compare(const nstime_t *a, const nstime_t *b);
|
||||
void nstime_add(nstime_t *time, const nstime_t *addend);
|
||||
void nstime_subtract(nstime_t *time, const nstime_t *subtrahend);
|
||||
void nstime_imultiply(nstime_t *time, uint64_t multiplier);
|
||||
void nstime_idivide(nstime_t *time, uint64_t divisor);
|
||||
uint64_t nstime_divide(const nstime_t *time, const nstime_t *divisor);
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef bool (nstime_monotonic_t)(void);
|
||||
extern nstime_monotonic_t *nstime_monotonic;
|
||||
typedef bool (nstime_update_t)(nstime_t *);
|
||||
extern nstime_update_t *nstime_update;
|
||||
#else
|
||||
bool nstime_monotonic(void);
|
||||
bool nstime_update(nstime_t *time);
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
@@ -1,29 +0,0 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
void *pages_map(void *addr, size_t size, bool *commit);
|
||||
void pages_unmap(void *addr, size_t size);
|
||||
void *pages_trim(void *addr, size_t alloc_size, size_t leadsize,
|
||||
size_t size, bool *commit);
|
||||
bool pages_commit(void *addr, size_t size);
|
||||
bool pages_decommit(void *addr, size_t size);
|
||||
bool pages_purge(void *addr, size_t size);
|
||||
bool pages_huge(void *addr, size_t size);
|
||||
bool pages_nohuge(void *addr, size_t size);
|
||||
void pages_boot(void);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
|
||||
@@ -1,345 +0,0 @@
|
||||
/*
|
||||
* A Pairing Heap implementation.
|
||||
*
|
||||
* "The Pairing Heap: A New Form of Self-Adjusting Heap"
|
||||
* https://www.cs.cmu.edu/~sleator/papers/pairing-heaps.pdf
|
||||
*
|
||||
* With auxiliary twopass list, described in a follow on paper.
|
||||
*
|
||||
* "Pairing Heaps: Experiments and Analysis"
|
||||
* http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.106.2988&rep=rep1&type=pdf
|
||||
*
|
||||
*******************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef PH_H_
|
||||
#define PH_H_
|
||||
|
||||
/* Node structure. */
|
||||
#define phn(a_type) \
|
||||
struct { \
|
||||
a_type *phn_prev; \
|
||||
a_type *phn_next; \
|
||||
a_type *phn_lchild; \
|
||||
}
|
||||
|
||||
/* Root structure. */
|
||||
#define ph(a_type) \
|
||||
struct { \
|
||||
a_type *ph_root; \
|
||||
}
|
||||
|
||||
/* Internal utility macros. */
|
||||
#define phn_lchild_get(a_type, a_field, a_phn) \
|
||||
(a_phn->a_field.phn_lchild)
|
||||
#define phn_lchild_set(a_type, a_field, a_phn, a_lchild) do { \
|
||||
a_phn->a_field.phn_lchild = a_lchild; \
|
||||
} while (0)
|
||||
|
||||
#define phn_next_get(a_type, a_field, a_phn) \
|
||||
(a_phn->a_field.phn_next)
|
||||
#define phn_prev_set(a_type, a_field, a_phn, a_prev) do { \
|
||||
a_phn->a_field.phn_prev = a_prev; \
|
||||
} while (0)
|
||||
|
||||
#define phn_prev_get(a_type, a_field, a_phn) \
|
||||
(a_phn->a_field.phn_prev)
|
||||
#define phn_next_set(a_type, a_field, a_phn, a_next) do { \
|
||||
a_phn->a_field.phn_next = a_next; \
|
||||
} while (0)
|
||||
|
||||
#define phn_merge_ordered(a_type, a_field, a_phn0, a_phn1, a_cmp) do { \
|
||||
a_type *phn0child; \
|
||||
\
|
||||
assert(a_phn0 != NULL); \
|
||||
assert(a_phn1 != NULL); \
|
||||
assert(a_cmp(a_phn0, a_phn1) <= 0); \
|
||||
\
|
||||
phn_prev_set(a_type, a_field, a_phn1, a_phn0); \
|
||||
phn0child = phn_lchild_get(a_type, a_field, a_phn0); \
|
||||
phn_next_set(a_type, a_field, a_phn1, phn0child); \
|
||||
if (phn0child != NULL) \
|
||||
phn_prev_set(a_type, a_field, phn0child, a_phn1); \
|
||||
phn_lchild_set(a_type, a_field, a_phn0, a_phn1); \
|
||||
} while (0)
|
||||
|
||||
#define phn_merge(a_type, a_field, a_phn0, a_phn1, a_cmp, r_phn) do { \
|
||||
if (a_phn0 == NULL) \
|
||||
r_phn = a_phn1; \
|
||||
else if (a_phn1 == NULL) \
|
||||
r_phn = a_phn0; \
|
||||
else if (a_cmp(a_phn0, a_phn1) < 0) { \
|
||||
phn_merge_ordered(a_type, a_field, a_phn0, a_phn1, \
|
||||
a_cmp); \
|
||||
r_phn = a_phn0; \
|
||||
} else { \
|
||||
phn_merge_ordered(a_type, a_field, a_phn1, a_phn0, \
|
||||
a_cmp); \
|
||||
r_phn = a_phn1; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define ph_merge_siblings(a_type, a_field, a_phn, a_cmp, r_phn) do { \
|
||||
a_type *head = NULL; \
|
||||
a_type *tail = NULL; \
|
||||
a_type *phn0 = a_phn; \
|
||||
a_type *phn1 = phn_next_get(a_type, a_field, phn0); \
|
||||
\
|
||||
/* \
|
||||
* Multipass merge, wherein the first two elements of a FIFO \
|
||||
* are repeatedly merged, and each result is appended to the \
|
||||
* singly linked FIFO, until the FIFO contains only a single \
|
||||
* element. We start with a sibling list but no reference to \
|
||||
* its tail, so we do a single pass over the sibling list to \
|
||||
* populate the FIFO. \
|
||||
*/ \
|
||||
if (phn1 != NULL) { \
|
||||
a_type *phnrest = phn_next_get(a_type, a_field, phn1); \
|
||||
if (phnrest != NULL) \
|
||||
phn_prev_set(a_type, a_field, phnrest, NULL); \
|
||||
phn_prev_set(a_type, a_field, phn0, NULL); \
|
||||
phn_next_set(a_type, a_field, phn0, NULL); \
|
||||
phn_prev_set(a_type, a_field, phn1, NULL); \
|
||||
phn_next_set(a_type, a_field, phn1, NULL); \
|
||||
phn_merge(a_type, a_field, phn0, phn1, a_cmp, phn0); \
|
||||
head = tail = phn0; \
|
||||
phn0 = phnrest; \
|
||||
while (phn0 != NULL) { \
|
||||
phn1 = phn_next_get(a_type, a_field, phn0); \
|
||||
if (phn1 != NULL) { \
|
||||
phnrest = phn_next_get(a_type, a_field, \
|
||||
phn1); \
|
||||
if (phnrest != NULL) { \
|
||||
phn_prev_set(a_type, a_field, \
|
||||
phnrest, NULL); \
|
||||
} \
|
||||
phn_prev_set(a_type, a_field, phn0, \
|
||||
NULL); \
|
||||
phn_next_set(a_type, a_field, phn0, \
|
||||
NULL); \
|
||||
phn_prev_set(a_type, a_field, phn1, \
|
||||
NULL); \
|
||||
phn_next_set(a_type, a_field, phn1, \
|
||||
NULL); \
|
||||
phn_merge(a_type, a_field, phn0, phn1, \
|
||||
a_cmp, phn0); \
|
||||
phn_next_set(a_type, a_field, tail, \
|
||||
phn0); \
|
||||
tail = phn0; \
|
||||
phn0 = phnrest; \
|
||||
} else { \
|
||||
phn_next_set(a_type, a_field, tail, \
|
||||
phn0); \
|
||||
tail = phn0; \
|
||||
phn0 = NULL; \
|
||||
} \
|
||||
} \
|
||||
phn0 = head; \
|
||||
phn1 = phn_next_get(a_type, a_field, phn0); \
|
||||
if (phn1 != NULL) { \
|
||||
while (true) { \
|
||||
head = phn_next_get(a_type, a_field, \
|
||||
phn1); \
|
||||
assert(phn_prev_get(a_type, a_field, \
|
||||
phn0) == NULL); \
|
||||
phn_next_set(a_type, a_field, phn0, \
|
||||
NULL); \
|
||||
assert(phn_prev_get(a_type, a_field, \
|
||||
phn1) == NULL); \
|
||||
phn_next_set(a_type, a_field, phn1, \
|
||||
NULL); \
|
||||
phn_merge(a_type, a_field, phn0, phn1, \
|
||||
a_cmp, phn0); \
|
||||
if (head == NULL) \
|
||||
break; \
|
||||
phn_next_set(a_type, a_field, tail, \
|
||||
phn0); \
|
||||
tail = phn0; \
|
||||
phn0 = head; \
|
||||
phn1 = phn_next_get(a_type, a_field, \
|
||||
phn0); \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
r_phn = phn0; \
|
||||
} while (0)
|
||||
|
||||
#define ph_merge_aux(a_type, a_field, a_ph, a_cmp) do { \
|
||||
a_type *phn = phn_next_get(a_type, a_field, a_ph->ph_root); \
|
||||
if (phn != NULL) { \
|
||||
phn_prev_set(a_type, a_field, a_ph->ph_root, NULL); \
|
||||
phn_next_set(a_type, a_field, a_ph->ph_root, NULL); \
|
||||
phn_prev_set(a_type, a_field, phn, NULL); \
|
||||
ph_merge_siblings(a_type, a_field, phn, a_cmp, phn); \
|
||||
assert(phn_next_get(a_type, a_field, phn) == NULL); \
|
||||
phn_merge(a_type, a_field, a_ph->ph_root, phn, a_cmp, \
|
||||
a_ph->ph_root); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define ph_merge_children(a_type, a_field, a_phn, a_cmp, r_phn) do { \
|
||||
a_type *lchild = phn_lchild_get(a_type, a_field, a_phn); \
|
||||
if (lchild == NULL) \
|
||||
r_phn = NULL; \
|
||||
else { \
|
||||
ph_merge_siblings(a_type, a_field, lchild, a_cmp, \
|
||||
r_phn); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/*
|
||||
* The ph_proto() macro generates function prototypes that correspond to the
|
||||
* functions generated by an equivalently parameterized call to ph_gen().
|
||||
*/
|
||||
#define ph_proto(a_attr, a_prefix, a_ph_type, a_type) \
|
||||
a_attr void a_prefix##new(a_ph_type *ph); \
|
||||
a_attr bool a_prefix##empty(a_ph_type *ph); \
|
||||
a_attr a_type *a_prefix##first(a_ph_type *ph); \
|
||||
a_attr void a_prefix##insert(a_ph_type *ph, a_type *phn); \
|
||||
a_attr a_type *a_prefix##remove_first(a_ph_type *ph); \
|
||||
a_attr void a_prefix##remove(a_ph_type *ph, a_type *phn);
|
||||
|
||||
/*
|
||||
* The ph_gen() macro generates a type-specific pairing heap implementation,
|
||||
* based on the above cpp macros.
|
||||
*/
|
||||
#define ph_gen(a_attr, a_prefix, a_ph_type, a_type, a_field, a_cmp) \
|
||||
a_attr void \
|
||||
a_prefix##new(a_ph_type *ph) \
|
||||
{ \
|
||||
\
|
||||
memset(ph, 0, sizeof(ph(a_type))); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_prefix##empty(a_ph_type *ph) \
|
||||
{ \
|
||||
\
|
||||
return (ph->ph_root == NULL); \
|
||||
} \
|
||||
a_attr a_type * \
|
||||
a_prefix##first(a_ph_type *ph) \
|
||||
{ \
|
||||
\
|
||||
if (ph->ph_root == NULL) \
|
||||
return (NULL); \
|
||||
ph_merge_aux(a_type, a_field, ph, a_cmp); \
|
||||
return (ph->ph_root); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_prefix##insert(a_ph_type *ph, a_type *phn) \
|
||||
{ \
|
||||
\
|
||||
memset(&phn->a_field, 0, sizeof(phn(a_type))); \
|
||||
\
|
||||
/* \
|
||||
* Treat the root as an aux list during insertion, and lazily \
|
||||
* merge during a_prefix##remove_first(). For elements that \
|
||||
* are inserted, then removed via a_prefix##remove() before the \
|
||||
* aux list is ever processed, this makes insert/remove \
|
||||
* constant-time, whereas eager merging would make insert \
|
||||
* O(log n). \
|
||||
*/ \
|
||||
if (ph->ph_root == NULL) \
|
||||
ph->ph_root = phn; \
|
||||
else { \
|
||||
phn_next_set(a_type, a_field, phn, phn_next_get(a_type, \
|
||||
a_field, ph->ph_root)); \
|
||||
if (phn_next_get(a_type, a_field, ph->ph_root) != \
|
||||
NULL) { \
|
||||
phn_prev_set(a_type, a_field, \
|
||||
phn_next_get(a_type, a_field, ph->ph_root), \
|
||||
phn); \
|
||||
} \
|
||||
phn_prev_set(a_type, a_field, phn, ph->ph_root); \
|
||||
phn_next_set(a_type, a_field, ph->ph_root, phn); \
|
||||
} \
|
||||
} \
|
||||
a_attr a_type * \
|
||||
a_prefix##remove_first(a_ph_type *ph) \
|
||||
{ \
|
||||
a_type *ret; \
|
||||
\
|
||||
if (ph->ph_root == NULL) \
|
||||
return (NULL); \
|
||||
ph_merge_aux(a_type, a_field, ph, a_cmp); \
|
||||
\
|
||||
ret = ph->ph_root; \
|
||||
\
|
||||
ph_merge_children(a_type, a_field, ph->ph_root, a_cmp, \
|
||||
ph->ph_root); \
|
||||
\
|
||||
return (ret); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_prefix##remove(a_ph_type *ph, a_type *phn) \
|
||||
{ \
|
||||
a_type *replace, *parent; \
|
||||
\
|
||||
/* \
|
||||
* We can delete from aux list without merging it, but we need \
|
||||
* to merge if we are dealing with the root node. \
|
||||
*/ \
|
||||
if (ph->ph_root == phn) { \
|
||||
ph_merge_aux(a_type, a_field, ph, a_cmp); \
|
||||
if (ph->ph_root == phn) { \
|
||||
ph_merge_children(a_type, a_field, ph->ph_root, \
|
||||
a_cmp, ph->ph_root); \
|
||||
return; \
|
||||
} \
|
||||
} \
|
||||
\
|
||||
/* Get parent (if phn is leftmost child) before mutating. */ \
|
||||
if ((parent = phn_prev_get(a_type, a_field, phn)) != NULL) { \
|
||||
if (phn_lchild_get(a_type, a_field, parent) != phn) \
|
||||
parent = NULL; \
|
||||
} \
|
||||
/* Find a possible replacement node, and link to parent. */ \
|
||||
ph_merge_children(a_type, a_field, phn, a_cmp, replace); \
|
||||
/* Set next/prev for sibling linked list. */ \
|
||||
if (replace != NULL) { \
|
||||
if (parent != NULL) { \
|
||||
phn_prev_set(a_type, a_field, replace, parent); \
|
||||
phn_lchild_set(a_type, a_field, parent, \
|
||||
replace); \
|
||||
} else { \
|
||||
phn_prev_set(a_type, a_field, replace, \
|
||||
phn_prev_get(a_type, a_field, phn)); \
|
||||
if (phn_prev_get(a_type, a_field, phn) != \
|
||||
NULL) { \
|
||||
phn_next_set(a_type, a_field, \
|
||||
phn_prev_get(a_type, a_field, phn), \
|
||||
replace); \
|
||||
} \
|
||||
} \
|
||||
phn_next_set(a_type, a_field, replace, \
|
||||
phn_next_get(a_type, a_field, phn)); \
|
||||
if (phn_next_get(a_type, a_field, phn) != NULL) { \
|
||||
phn_prev_set(a_type, a_field, \
|
||||
phn_next_get(a_type, a_field, phn), \
|
||||
replace); \
|
||||
} \
|
||||
} else { \
|
||||
if (parent != NULL) { \
|
||||
a_type *next = phn_next_get(a_type, a_field, \
|
||||
phn); \
|
||||
phn_lchild_set(a_type, a_field, parent, next); \
|
||||
if (next != NULL) { \
|
||||
phn_prev_set(a_type, a_field, next, \
|
||||
parent); \
|
||||
} \
|
||||
} else { \
|
||||
assert(phn_prev_get(a_type, a_field, phn) != \
|
||||
NULL); \
|
||||
phn_next_set(a_type, a_field, \
|
||||
phn_prev_get(a_type, a_field, phn), \
|
||||
phn_next_get(a_type, a_field, phn)); \
|
||||
} \
|
||||
if (phn_next_get(a_type, a_field, phn) != NULL) { \
|
||||
phn_prev_set(a_type, a_field, \
|
||||
phn_next_get(a_type, a_field, phn), \
|
||||
phn_prev_get(a_type, a_field, phn)); \
|
||||
} \
|
||||
} \
|
||||
}
|
||||
|
||||
#endif /* PH_H_ */
|
||||
@@ -1,199 +1,147 @@
|
||||
a0dalloc
|
||||
a0get
|
||||
a0calloc
|
||||
a0free
|
||||
a0malloc
|
||||
arena_aalloc
|
||||
arena_alloc_junk_small
|
||||
arena_basic_stats_merge
|
||||
arena_bin_index
|
||||
arena_bin_info
|
||||
arena_bitselm_get_const
|
||||
arena_bitselm_get_mutable
|
||||
arena_boot
|
||||
arena_choose
|
||||
arena_choose_hard
|
||||
arena_choose_impl
|
||||
arena_chunk_alloc_huge
|
||||
arena_chunk_cache_maybe_insert
|
||||
arena_chunk_cache_maybe_remove
|
||||
arena_chunk_dalloc_huge
|
||||
arena_chunk_ralloc_huge_expand
|
||||
arena_chunk_ralloc_huge_shrink
|
||||
arena_chunk_ralloc_huge_similar
|
||||
arena_cleanup
|
||||
arena_dalloc
|
||||
arena_dalloc_bin
|
||||
arena_dalloc_bin_junked_locked
|
||||
arena_dalloc_bin_locked
|
||||
arena_dalloc_junk_large
|
||||
arena_dalloc_junk_small
|
||||
arena_dalloc_large
|
||||
arena_dalloc_large_junked_locked
|
||||
arena_dalloc_large_locked
|
||||
arena_dalloc_small
|
||||
arena_decay_tick
|
||||
arena_decay_ticks
|
||||
arena_decay_time_default_get
|
||||
arena_decay_time_default_set
|
||||
arena_decay_time_get
|
||||
arena_decay_time_set
|
||||
arena_dss_prec_get
|
||||
arena_dss_prec_set
|
||||
arena_extent_sn_next
|
||||
arena_get
|
||||
arena_ichoose
|
||||
arena_init
|
||||
arena_lg_dirty_mult_default_get
|
||||
arena_lg_dirty_mult_default_set
|
||||
arena_lg_dirty_mult_get
|
||||
arena_lg_dirty_mult_set
|
||||
arena_malloc
|
||||
arena_malloc_hard
|
||||
arena_malloc_large
|
||||
arena_malloc_small
|
||||
arena_mapbits_allocated_get
|
||||
arena_mapbits_binind_get
|
||||
arena_mapbits_decommitted_get
|
||||
arena_mapbits_dirty_get
|
||||
arena_mapbits_get
|
||||
arena_mapbits_internal_set
|
||||
arena_mapbits_large_binind_set
|
||||
arena_mapbits_large_get
|
||||
arena_mapbits_large_set
|
||||
arena_mapbits_large_size_get
|
||||
arena_mapbits_size_decode
|
||||
arena_mapbits_size_encode
|
||||
arena_mapbits_small_runind_get
|
||||
arena_mapbits_small_set
|
||||
arena_mapbits_unallocated_set
|
||||
arena_mapbits_unallocated_size_get
|
||||
arena_mapbits_unallocated_size_set
|
||||
arena_mapbits_unzeroed_get
|
||||
arena_mapbitsp_get_const
|
||||
arena_mapbitsp_get_mutable
|
||||
arena_mapbits_unzeroed_set
|
||||
arena_mapbitsp_get
|
||||
arena_mapbitsp_read
|
||||
arena_mapbitsp_write
|
||||
arena_maxrun
|
||||
arena_maybe_purge
|
||||
arena_metadata_allocated_add
|
||||
arena_metadata_allocated_get
|
||||
arena_metadata_allocated_sub
|
||||
arena_migrate
|
||||
arena_miscelm_get_const
|
||||
arena_miscelm_get_mutable
|
||||
arena_miscelm_to_pageind
|
||||
arena_miscelm_to_rpages
|
||||
arena_mapp_get
|
||||
arena_maxclass
|
||||
arena_new
|
||||
arena_node_alloc
|
||||
arena_node_dalloc
|
||||
arena_nthreads_dec
|
||||
arena_nthreads_get
|
||||
arena_nthreads_inc
|
||||
arena_palloc
|
||||
arena_postfork_child
|
||||
arena_postfork_parent
|
||||
arena_prefork0
|
||||
arena_prefork1
|
||||
arena_prefork2
|
||||
arena_prefork3
|
||||
arena_prefork
|
||||
arena_prof_accum
|
||||
arena_prof_accum_impl
|
||||
arena_prof_accum_locked
|
||||
arena_prof_ctx_get
|
||||
arena_prof_ctx_set
|
||||
arena_prof_promoted
|
||||
arena_prof_tctx_get
|
||||
arena_prof_tctx_reset
|
||||
arena_prof_tctx_set
|
||||
arena_ptr_small_binind_get
|
||||
arena_purge
|
||||
arena_purge_all
|
||||
arena_quarantine_junk_small
|
||||
arena_ralloc
|
||||
arena_ralloc_junk_large
|
||||
arena_ralloc_no_move
|
||||
arena_rd_to_miscelm
|
||||
arena_redzone_corruption
|
||||
arena_reset
|
||||
arena_run_regind
|
||||
arena_run_to_miscelm
|
||||
arena_salloc
|
||||
arena_sdalloc
|
||||
arena_stats_merge
|
||||
arena_tcache_fill_small
|
||||
arena_tdata_get
|
||||
arena_tdata_get_hard
|
||||
arenas
|
||||
arenas_tdata_bypass_cleanup
|
||||
arenas_tdata_cleanup
|
||||
atomic_add_p
|
||||
arenas_booted
|
||||
arenas_cleanup
|
||||
arenas_extend
|
||||
arenas_initialized
|
||||
arenas_lock
|
||||
arenas_tls
|
||||
arenas_tsd
|
||||
arenas_tsd_boot
|
||||
arenas_tsd_cleanup_wrapper
|
||||
arenas_tsd_get
|
||||
arenas_tsd_get_wrapper
|
||||
arenas_tsd_init_head
|
||||
arenas_tsd_set
|
||||
atomic_add_u
|
||||
atomic_add_uint32
|
||||
atomic_add_uint64
|
||||
atomic_add_z
|
||||
atomic_cas_p
|
||||
atomic_cas_u
|
||||
atomic_cas_uint32
|
||||
atomic_cas_uint64
|
||||
atomic_cas_z
|
||||
atomic_sub_p
|
||||
atomic_sub_u
|
||||
atomic_sub_uint32
|
||||
atomic_sub_uint64
|
||||
atomic_sub_z
|
||||
atomic_write_p
|
||||
atomic_write_u
|
||||
atomic_write_uint32
|
||||
atomic_write_uint64
|
||||
atomic_write_z
|
||||
base_alloc
|
||||
base_boot
|
||||
base_calloc
|
||||
base_node_alloc
|
||||
base_node_dealloc
|
||||
base_postfork_child
|
||||
base_postfork_parent
|
||||
base_prefork
|
||||
base_stats_get
|
||||
bitmap_full
|
||||
bitmap_get
|
||||
bitmap_info_init
|
||||
bitmap_info_ngroups
|
||||
bitmap_init
|
||||
bitmap_set
|
||||
bitmap_sfu
|
||||
bitmap_size
|
||||
bitmap_unset
|
||||
bootstrap_calloc
|
||||
bootstrap_free
|
||||
bootstrap_malloc
|
||||
bt_init
|
||||
buferror
|
||||
chunk_alloc_base
|
||||
chunk_alloc_cache
|
||||
choose_arena
|
||||
choose_arena_hard
|
||||
chunk_alloc
|
||||
chunk_alloc_dss
|
||||
chunk_alloc_mmap
|
||||
chunk_alloc_wrapper
|
||||
chunk_boot
|
||||
chunk_dalloc_cache
|
||||
chunk_dalloc_mmap
|
||||
chunk_dalloc_wrapper
|
||||
chunk_deregister
|
||||
chunk_dealloc
|
||||
chunk_dealloc_mmap
|
||||
chunk_dss_boot
|
||||
chunk_dss_mergeable
|
||||
chunk_dss_postfork_child
|
||||
chunk_dss_postfork_parent
|
||||
chunk_dss_prec_get
|
||||
chunk_dss_prec_set
|
||||
chunk_hooks_default
|
||||
chunk_hooks_get
|
||||
chunk_hooks_set
|
||||
chunk_dss_prefork
|
||||
chunk_in_dss
|
||||
chunk_lookup
|
||||
chunk_npages
|
||||
chunk_purge_wrapper
|
||||
chunk_register
|
||||
chunk_postfork_child
|
||||
chunk_postfork_parent
|
||||
chunk_prefork
|
||||
chunk_unmap
|
||||
chunks_mtx
|
||||
chunks_rtree
|
||||
chunksize
|
||||
chunksize_mask
|
||||
ckh_bucket_search
|
||||
ckh_count
|
||||
ckh_delete
|
||||
ckh_evict_reloc_insert
|
||||
ckh_insert
|
||||
ckh_isearch
|
||||
ckh_iter
|
||||
ckh_new
|
||||
ckh_pointer_hash
|
||||
ckh_pointer_keycomp
|
||||
ckh_rebuild
|
||||
ckh_remove
|
||||
ckh_search
|
||||
ckh_string_hash
|
||||
ckh_string_keycomp
|
||||
ckh_try_bucket_insert
|
||||
ckh_try_insert
|
||||
ctl_boot
|
||||
ctl_bymib
|
||||
ctl_byname
|
||||
@@ -201,31 +149,7 @@ ctl_nametomib
|
||||
ctl_postfork_child
|
||||
ctl_postfork_parent
|
||||
ctl_prefork
|
||||
decay_ticker_get
|
||||
dss_prec_names
|
||||
extent_node_achunk_get
|
||||
extent_node_achunk_set
|
||||
extent_node_addr_get
|
||||
extent_node_addr_set
|
||||
extent_node_arena_get
|
||||
extent_node_arena_set
|
||||
extent_node_committed_get
|
||||
extent_node_committed_set
|
||||
extent_node_dirty_insert
|
||||
extent_node_dirty_linkage_init
|
||||
extent_node_dirty_remove
|
||||
extent_node_init
|
||||
extent_node_prof_tctx_get
|
||||
extent_node_prof_tctx_set
|
||||
extent_node_size_get
|
||||
extent_node_size_set
|
||||
extent_node_sn_get
|
||||
extent_node_sn_set
|
||||
extent_node_zeroed_get
|
||||
extent_node_zeroed_set
|
||||
extent_tree_ad_destroy
|
||||
extent_tree_ad_destroy_recurse
|
||||
extent_tree_ad_empty
|
||||
extent_tree_ad_first
|
||||
extent_tree_ad_insert
|
||||
extent_tree_ad_iter
|
||||
@@ -242,31 +166,22 @@ extent_tree_ad_reverse_iter
|
||||
extent_tree_ad_reverse_iter_recurse
|
||||
extent_tree_ad_reverse_iter_start
|
||||
extent_tree_ad_search
|
||||
extent_tree_szsnad_destroy
|
||||
extent_tree_szsnad_destroy_recurse
|
||||
extent_tree_szsnad_empty
|
||||
extent_tree_szsnad_first
|
||||
extent_tree_szsnad_insert
|
||||
extent_tree_szsnad_iter
|
||||
extent_tree_szsnad_iter_recurse
|
||||
extent_tree_szsnad_iter_start
|
||||
extent_tree_szsnad_last
|
||||
extent_tree_szsnad_new
|
||||
extent_tree_szsnad_next
|
||||
extent_tree_szsnad_nsearch
|
||||
extent_tree_szsnad_prev
|
||||
extent_tree_szsnad_psearch
|
||||
extent_tree_szsnad_remove
|
||||
extent_tree_szsnad_reverse_iter
|
||||
extent_tree_szsnad_reverse_iter_recurse
|
||||
extent_tree_szsnad_reverse_iter_start
|
||||
extent_tree_szsnad_search
|
||||
ffs_llu
|
||||
ffs_lu
|
||||
ffs_u
|
||||
ffs_u32
|
||||
ffs_u64
|
||||
ffs_zu
|
||||
extent_tree_szad_first
|
||||
extent_tree_szad_insert
|
||||
extent_tree_szad_iter
|
||||
extent_tree_szad_iter_recurse
|
||||
extent_tree_szad_iter_start
|
||||
extent_tree_szad_last
|
||||
extent_tree_szad_new
|
||||
extent_tree_szad_next
|
||||
extent_tree_szad_nsearch
|
||||
extent_tree_szad_prev
|
||||
extent_tree_szad_psearch
|
||||
extent_tree_szad_remove
|
||||
extent_tree_szad_reverse_iter
|
||||
extent_tree_szad_reverse_iter_recurse
|
||||
extent_tree_szad_reverse_iter_start
|
||||
extent_tree_szad_search
|
||||
get_errno
|
||||
hash
|
||||
hash_fmix_32
|
||||
@@ -278,51 +193,46 @@ hash_rotl_64
|
||||
hash_x64_128
|
||||
hash_x86_128
|
||||
hash_x86_32
|
||||
huge_aalloc
|
||||
huge_allocated
|
||||
huge_boot
|
||||
huge_dalloc
|
||||
huge_dalloc_junk
|
||||
huge_dss_prec_get
|
||||
huge_malloc
|
||||
huge_mtx
|
||||
huge_ndalloc
|
||||
huge_nmalloc
|
||||
huge_palloc
|
||||
huge_prof_tctx_get
|
||||
huge_prof_tctx_reset
|
||||
huge_prof_tctx_set
|
||||
huge_postfork_child
|
||||
huge_postfork_parent
|
||||
huge_prefork
|
||||
huge_prof_ctx_get
|
||||
huge_prof_ctx_set
|
||||
huge_ralloc
|
||||
huge_ralloc_no_move
|
||||
huge_salloc
|
||||
iaalloc
|
||||
ialloc
|
||||
iallocztm
|
||||
iarena_cleanup
|
||||
iallocm
|
||||
icalloc
|
||||
icalloct
|
||||
idalloc
|
||||
idalloctm
|
||||
in_valgrind
|
||||
index2size
|
||||
index2size_compute
|
||||
index2size_lookup
|
||||
index2size_tab
|
||||
idalloct
|
||||
imalloc
|
||||
imalloct
|
||||
ipalloc
|
||||
ipalloct
|
||||
ipallocztm
|
||||
iqalloc
|
||||
iqalloct
|
||||
iralloc
|
||||
iralloct
|
||||
iralloct_realign
|
||||
isalloc
|
||||
isdalloct
|
||||
isqalloc
|
||||
isthreaded
|
||||
ivsalloc
|
||||
ixalloc
|
||||
jemalloc_postfork_child
|
||||
jemalloc_postfork_parent
|
||||
jemalloc_prefork
|
||||
large_maxclass
|
||||
lg_floor
|
||||
lg_prof_sample
|
||||
malloc_cprintf
|
||||
malloc_mutex_assert_not_owner
|
||||
malloc_mutex_assert_owner
|
||||
malloc_mutex_boot
|
||||
malloc_mutex_init
|
||||
malloc_mutex_lock
|
||||
malloc_mutex_postfork_child
|
||||
@@ -332,8 +242,7 @@ malloc_mutex_unlock
|
||||
malloc_printf
|
||||
malloc_snprintf
|
||||
malloc_strtoumax
|
||||
malloc_tsd_boot0
|
||||
malloc_tsd_boot1
|
||||
malloc_tsd_boot
|
||||
malloc_tsd_cleanup_register
|
||||
malloc_tsd_dalloc
|
||||
malloc_tsd_malloc
|
||||
@@ -342,35 +251,16 @@ malloc_vcprintf
|
||||
malloc_vsnprintf
|
||||
malloc_write
|
||||
map_bias
|
||||
map_misc_offset
|
||||
mb_write
|
||||
mutex_boot
|
||||
narenas_auto
|
||||
narenas_tdata_cleanup
|
||||
narenas_total
|
||||
narenas_total_get
|
||||
ncpus
|
||||
nhbins
|
||||
nhclasses
|
||||
nlclasses
|
||||
nstime_add
|
||||
nstime_compare
|
||||
nstime_copy
|
||||
nstime_divide
|
||||
nstime_idivide
|
||||
nstime_imultiply
|
||||
nstime_init
|
||||
nstime_init2
|
||||
nstime_monotonic
|
||||
nstime_ns
|
||||
nstime_nsec
|
||||
nstime_sec
|
||||
nstime_subtract
|
||||
nstime_update
|
||||
opt_abort
|
||||
opt_decay_time
|
||||
opt_dss
|
||||
opt_junk
|
||||
opt_junk_alloc
|
||||
opt_junk_free
|
||||
opt_lg_chunk
|
||||
opt_lg_dirty_mult
|
||||
opt_lg_prof_interval
|
||||
@@ -384,248 +274,140 @@ opt_prof_final
|
||||
opt_prof_gdump
|
||||
opt_prof_leak
|
||||
opt_prof_prefix
|
||||
opt_prof_thread_active_init
|
||||
opt_purge
|
||||
opt_quarantine
|
||||
opt_redzone
|
||||
opt_stats_print
|
||||
opt_tcache
|
||||
opt_utrace
|
||||
opt_valgrind
|
||||
opt_xmalloc
|
||||
opt_zero
|
||||
p2rz
|
||||
pages_boot
|
||||
pages_commit
|
||||
pages_decommit
|
||||
pages_huge
|
||||
pages_map
|
||||
pages_nohuge
|
||||
pages_purge
|
||||
pages_trim
|
||||
pages_unmap
|
||||
pind2sz
|
||||
pind2sz_compute
|
||||
pind2sz_lookup
|
||||
pind2sz_tab
|
||||
pow2_ceil_u32
|
||||
pow2_ceil_u64
|
||||
pow2_ceil_zu
|
||||
prng_lg_range_u32
|
||||
prng_lg_range_u64
|
||||
prng_lg_range_zu
|
||||
prng_range_u32
|
||||
prng_range_u64
|
||||
prng_range_zu
|
||||
prng_state_next_u32
|
||||
prng_state_next_u64
|
||||
prng_state_next_zu
|
||||
prof_active
|
||||
prof_active_get
|
||||
prof_active_get_unlocked
|
||||
prof_active_set
|
||||
prof_alloc_prep
|
||||
prof_alloc_rollback
|
||||
pow2_ceil
|
||||
prof_backtrace
|
||||
prof_boot0
|
||||
prof_boot1
|
||||
prof_boot2
|
||||
prof_bt_count
|
||||
prof_dump_header
|
||||
prof_ctx_get
|
||||
prof_ctx_set
|
||||
prof_dump_open
|
||||
prof_free
|
||||
prof_free_sampled_object
|
||||
prof_gdump
|
||||
prof_gdump_get
|
||||
prof_gdump_get_unlocked
|
||||
prof_gdump_set
|
||||
prof_gdump_val
|
||||
prof_idump
|
||||
prof_interval
|
||||
prof_lookup
|
||||
prof_malloc
|
||||
prof_malloc_sample_object
|
||||
prof_mdump
|
||||
prof_postfork_child
|
||||
prof_postfork_parent
|
||||
prof_prefork0
|
||||
prof_prefork1
|
||||
prof_prefork
|
||||
prof_promote
|
||||
prof_realloc
|
||||
prof_reset
|
||||
prof_sample_accum_update
|
||||
prof_sample_threshold_update
|
||||
prof_tctx_get
|
||||
prof_tctx_reset
|
||||
prof_tctx_set
|
||||
prof_tdata_booted
|
||||
prof_tdata_cleanup
|
||||
prof_tdata_count
|
||||
prof_tdata_get
|
||||
prof_tdata_init
|
||||
prof_tdata_reinit
|
||||
prof_thread_active_get
|
||||
prof_thread_active_init_get
|
||||
prof_thread_active_init_set
|
||||
prof_thread_active_set
|
||||
prof_thread_name_get
|
||||
prof_thread_name_set
|
||||
psz2ind
|
||||
psz2u
|
||||
purge_mode_names
|
||||
prof_tdata_initialized
|
||||
prof_tdata_tls
|
||||
prof_tdata_tsd
|
||||
prof_tdata_tsd_boot
|
||||
prof_tdata_tsd_cleanup_wrapper
|
||||
prof_tdata_tsd_get
|
||||
prof_tdata_tsd_get_wrapper
|
||||
prof_tdata_tsd_init_head
|
||||
prof_tdata_tsd_set
|
||||
quarantine
|
||||
quarantine_alloc_hook
|
||||
quarantine_alloc_hook_work
|
||||
quarantine_boot
|
||||
quarantine_booted
|
||||
quarantine_cleanup
|
||||
rtree_child_read
|
||||
rtree_child_read_hard
|
||||
rtree_child_tryread
|
||||
quarantine_init
|
||||
quarantine_tls
|
||||
quarantine_tsd
|
||||
quarantine_tsd_boot
|
||||
quarantine_tsd_cleanup_wrapper
|
||||
quarantine_tsd_get
|
||||
quarantine_tsd_get_wrapper
|
||||
quarantine_tsd_init_head
|
||||
quarantine_tsd_set
|
||||
register_zone
|
||||
rtree_delete
|
||||
rtree_get
|
||||
rtree_get_locked
|
||||
rtree_new
|
||||
rtree_node_valid
|
||||
rtree_postfork_child
|
||||
rtree_postfork_parent
|
||||
rtree_prefork
|
||||
rtree_set
|
||||
rtree_start_level
|
||||
rtree_subkey
|
||||
rtree_subtree_read
|
||||
rtree_subtree_read_hard
|
||||
rtree_subtree_tryread
|
||||
rtree_val_read
|
||||
rtree_val_write
|
||||
run_quantize_ceil
|
||||
run_quantize_floor
|
||||
s2u
|
||||
s2u_compute
|
||||
s2u_lookup
|
||||
sa2u
|
||||
set_errno
|
||||
size2index
|
||||
size2index_compute
|
||||
size2index_lookup
|
||||
size2index_tab
|
||||
spin_adaptive
|
||||
spin_init
|
||||
small_size2bin
|
||||
stats_cactive
|
||||
stats_cactive_add
|
||||
stats_cactive_get
|
||||
stats_cactive_sub
|
||||
stats_chunks
|
||||
stats_print
|
||||
tcache_alloc_easy
|
||||
tcache_alloc_large
|
||||
tcache_alloc_small
|
||||
tcache_alloc_small_hard
|
||||
tcache_arena_reassociate
|
||||
tcache_arena_associate
|
||||
tcache_arena_dissociate
|
||||
tcache_bin_flush_large
|
||||
tcache_bin_flush_small
|
||||
tcache_bin_info
|
||||
tcache_boot
|
||||
tcache_cleanup
|
||||
tcache_boot0
|
||||
tcache_boot1
|
||||
tcache_booted
|
||||
tcache_create
|
||||
tcache_dalloc_large
|
||||
tcache_dalloc_small
|
||||
tcache_enabled_cleanup
|
||||
tcache_destroy
|
||||
tcache_enabled_booted
|
||||
tcache_enabled_get
|
||||
tcache_enabled_initialized
|
||||
tcache_enabled_set
|
||||
tcache_enabled_tls
|
||||
tcache_enabled_tsd
|
||||
tcache_enabled_tsd_boot
|
||||
tcache_enabled_tsd_cleanup_wrapper
|
||||
tcache_enabled_tsd_get
|
||||
tcache_enabled_tsd_get_wrapper
|
||||
tcache_enabled_tsd_init_head
|
||||
tcache_enabled_tsd_set
|
||||
tcache_event
|
||||
tcache_event_hard
|
||||
tcache_flush
|
||||
tcache_get
|
||||
tcache_get_hard
|
||||
tcache_initialized
|
||||
tcache_maxclass
|
||||
tcache_salloc
|
||||
tcache_stats_merge
|
||||
tcaches
|
||||
tcaches_create
|
||||
tcaches_destroy
|
||||
tcaches_flush
|
||||
tcaches_get
|
||||
thread_allocated_cleanup
|
||||
thread_deallocated_cleanup
|
||||
ticker_copy
|
||||
ticker_init
|
||||
ticker_read
|
||||
ticker_tick
|
||||
ticker_ticks
|
||||
tsd_arena_get
|
||||
tsd_arena_set
|
||||
tsd_arenap_get
|
||||
tsd_arenas_tdata_bypass_get
|
||||
tsd_arenas_tdata_bypass_set
|
||||
tsd_arenas_tdata_bypassp_get
|
||||
tsd_arenas_tdata_get
|
||||
tsd_arenas_tdata_set
|
||||
tsd_arenas_tdatap_get
|
||||
tsd_boot
|
||||
tsd_boot0
|
||||
tsd_boot1
|
||||
tsd_booted
|
||||
tsd_booted_get
|
||||
tsd_cleanup
|
||||
tsd_cleanup_wrapper
|
||||
tsd_fetch
|
||||
tsd_fetch_impl
|
||||
tsd_get
|
||||
tsd_get_allocates
|
||||
tsd_iarena_get
|
||||
tsd_iarena_set
|
||||
tsd_iarenap_get
|
||||
tsd_initialized
|
||||
tcache_thread_cleanup
|
||||
tcache_tls
|
||||
tcache_tsd
|
||||
tcache_tsd_boot
|
||||
tcache_tsd_cleanup_wrapper
|
||||
tcache_tsd_get
|
||||
tcache_tsd_get_wrapper
|
||||
tcache_tsd_init_head
|
||||
tcache_tsd_set
|
||||
thread_allocated_booted
|
||||
thread_allocated_initialized
|
||||
thread_allocated_tls
|
||||
thread_allocated_tsd
|
||||
thread_allocated_tsd_boot
|
||||
thread_allocated_tsd_cleanup_wrapper
|
||||
thread_allocated_tsd_get
|
||||
thread_allocated_tsd_get_wrapper
|
||||
thread_allocated_tsd_init_head
|
||||
thread_allocated_tsd_set
|
||||
tsd_init_check_recursion
|
||||
tsd_init_finish
|
||||
tsd_init_head
|
||||
tsd_narenas_tdata_get
|
||||
tsd_narenas_tdata_set
|
||||
tsd_narenas_tdatap_get
|
||||
tsd_wrapper_get
|
||||
tsd_wrapper_set
|
||||
tsd_nominal
|
||||
tsd_prof_tdata_get
|
||||
tsd_prof_tdata_set
|
||||
tsd_prof_tdatap_get
|
||||
tsd_quarantine_get
|
||||
tsd_quarantine_set
|
||||
tsd_quarantinep_get
|
||||
tsd_set
|
||||
tsd_tcache_enabled_get
|
||||
tsd_tcache_enabled_set
|
||||
tsd_tcache_enabledp_get
|
||||
tsd_tcache_get
|
||||
tsd_tcache_set
|
||||
tsd_tcachep_get
|
||||
tsd_thread_allocated_get
|
||||
tsd_thread_allocated_set
|
||||
tsd_thread_allocatedp_get
|
||||
tsd_thread_deallocated_get
|
||||
tsd_thread_deallocated_set
|
||||
tsd_thread_deallocatedp_get
|
||||
tsd_tls
|
||||
tsd_tsd
|
||||
tsd_tsdn
|
||||
tsd_witness_fork_get
|
||||
tsd_witness_fork_set
|
||||
tsd_witness_forkp_get
|
||||
tsd_witnesses_get
|
||||
tsd_witnesses_set
|
||||
tsd_witnessesp_get
|
||||
tsdn_fetch
|
||||
tsdn_null
|
||||
tsdn_tsd
|
||||
u2rz
|
||||
valgrind_freelike_block
|
||||
valgrind_make_mem_defined
|
||||
valgrind_make_mem_noaccess
|
||||
valgrind_make_mem_undefined
|
||||
witness_assert_lockless
|
||||
witness_assert_not_owner
|
||||
witness_assert_owner
|
||||
witness_fork_cleanup
|
||||
witness_init
|
||||
witness_lock
|
||||
witness_lock_error
|
||||
witness_lockless_error
|
||||
witness_not_owner_error
|
||||
witness_owner
|
||||
witness_owner_error
|
||||
witness_postfork_child
|
||||
witness_postfork_parent
|
||||
witness_prefork
|
||||
witness_unlock
|
||||
witnesses_cleanup
|
||||
zone_register
|
||||
|
||||
@@ -15,16 +15,34 @@
|
||||
* See Knuth's TAOCP 3rd Ed., Vol. 2, pg. 17 for details on these constraints.
|
||||
*
|
||||
* This choice of m has the disadvantage that the quality of the bits is
|
||||
* proportional to bit position. For example, the lowest bit has a cycle of 2,
|
||||
* proportional to bit position. For example. the lowest bit has a cycle of 2,
|
||||
* the next has a cycle of 4, etc. For this reason, we prefer to use the upper
|
||||
* bits.
|
||||
*
|
||||
* Macro parameters:
|
||||
* uint32_t r : Result.
|
||||
* unsigned lg_range : (0..32], number of least significant bits to return.
|
||||
* uint32_t state : Seed value.
|
||||
* const uint32_t a, c : See above discussion.
|
||||
*/
|
||||
#define prng32(r, lg_range, state, a, c) do { \
|
||||
assert(lg_range > 0); \
|
||||
assert(lg_range <= 32); \
|
||||
\
|
||||
r = (state * (a)) + (c); \
|
||||
state = r; \
|
||||
r >>= (32 - lg_range); \
|
||||
} while (false)
|
||||
|
||||
#define PRNG_A_32 UINT32_C(1103515241)
|
||||
#define PRNG_C_32 UINT32_C(12347)
|
||||
|
||||
#define PRNG_A_64 UINT64_C(6364136223846793005)
|
||||
#define PRNG_C_64 UINT64_C(1442695040888963407)
|
||||
/* Same as prng32(), but 64 bits of pseudo-randomness, using uint64_t. */
|
||||
#define prng64(r, lg_range, state, a, c) do { \
|
||||
assert(lg_range > 0); \
|
||||
assert(lg_range <= 64); \
|
||||
\
|
||||
r = (state * (a)) + (c); \
|
||||
state = r; \
|
||||
r >>= (64 - lg_range); \
|
||||
} while (false)
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
@@ -38,170 +56,5 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
uint32_t prng_state_next_u32(uint32_t state);
|
||||
uint64_t prng_state_next_u64(uint64_t state);
|
||||
size_t prng_state_next_zu(size_t state);
|
||||
|
||||
uint32_t prng_lg_range_u32(uint32_t *state, unsigned lg_range,
|
||||
bool atomic);
|
||||
uint64_t prng_lg_range_u64(uint64_t *state, unsigned lg_range);
|
||||
size_t prng_lg_range_zu(size_t *state, unsigned lg_range, bool atomic);
|
||||
|
||||
uint32_t prng_range_u32(uint32_t *state, uint32_t range, bool atomic);
|
||||
uint64_t prng_range_u64(uint64_t *state, uint64_t range);
|
||||
size_t prng_range_zu(size_t *state, size_t range, bool atomic);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PRNG_C_))
|
||||
JEMALLOC_ALWAYS_INLINE uint32_t
|
||||
prng_state_next_u32(uint32_t state)
|
||||
{
|
||||
|
||||
return ((state * PRNG_A_32) + PRNG_C_32);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE uint64_t
|
||||
prng_state_next_u64(uint64_t state)
|
||||
{
|
||||
|
||||
return ((state * PRNG_A_64) + PRNG_C_64);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE size_t
|
||||
prng_state_next_zu(size_t state)
|
||||
{
|
||||
|
||||
#if LG_SIZEOF_PTR == 2
|
||||
return ((state * PRNG_A_32) + PRNG_C_32);
|
||||
#elif LG_SIZEOF_PTR == 3
|
||||
return ((state * PRNG_A_64) + PRNG_C_64);
|
||||
#else
|
||||
#error Unsupported pointer size
|
||||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE uint32_t
|
||||
prng_lg_range_u32(uint32_t *state, unsigned lg_range, bool atomic)
|
||||
{
|
||||
uint32_t ret, state1;
|
||||
|
||||
assert(lg_range > 0);
|
||||
assert(lg_range <= 32);
|
||||
|
||||
if (atomic) {
|
||||
uint32_t state0;
|
||||
|
||||
do {
|
||||
state0 = atomic_read_uint32(state);
|
||||
state1 = prng_state_next_u32(state0);
|
||||
} while (atomic_cas_uint32(state, state0, state1));
|
||||
} else {
|
||||
state1 = prng_state_next_u32(*state);
|
||||
*state = state1;
|
||||
}
|
||||
ret = state1 >> (32 - lg_range);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
/* 64-bit atomic operations cannot be supported on all relevant platforms. */
|
||||
JEMALLOC_ALWAYS_INLINE uint64_t
|
||||
prng_lg_range_u64(uint64_t *state, unsigned lg_range)
|
||||
{
|
||||
uint64_t ret, state1;
|
||||
|
||||
assert(lg_range > 0);
|
||||
assert(lg_range <= 64);
|
||||
|
||||
state1 = prng_state_next_u64(*state);
|
||||
*state = state1;
|
||||
ret = state1 >> (64 - lg_range);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE size_t
|
||||
prng_lg_range_zu(size_t *state, unsigned lg_range, bool atomic)
|
||||
{
|
||||
size_t ret, state1;
|
||||
|
||||
assert(lg_range > 0);
|
||||
assert(lg_range <= ZU(1) << (3 + LG_SIZEOF_PTR));
|
||||
|
||||
if (atomic) {
|
||||
size_t state0;
|
||||
|
||||
do {
|
||||
state0 = atomic_read_z(state);
|
||||
state1 = prng_state_next_zu(state0);
|
||||
} while (atomic_cas_z(state, state0, state1));
|
||||
} else {
|
||||
state1 = prng_state_next_zu(*state);
|
||||
*state = state1;
|
||||
}
|
||||
ret = state1 >> ((ZU(1) << (3 + LG_SIZEOF_PTR)) - lg_range);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE uint32_t
|
||||
prng_range_u32(uint32_t *state, uint32_t range, bool atomic)
|
||||
{
|
||||
uint32_t ret;
|
||||
unsigned lg_range;
|
||||
|
||||
assert(range > 1);
|
||||
|
||||
/* Compute the ceiling of lg(range). */
|
||||
lg_range = ffs_u32(pow2_ceil_u32(range)) - 1;
|
||||
|
||||
/* Generate a result in [0..range) via repeated trial. */
|
||||
do {
|
||||
ret = prng_lg_range_u32(state, lg_range, atomic);
|
||||
} while (ret >= range);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE uint64_t
|
||||
prng_range_u64(uint64_t *state, uint64_t range)
|
||||
{
|
||||
uint64_t ret;
|
||||
unsigned lg_range;
|
||||
|
||||
assert(range > 1);
|
||||
|
||||
/* Compute the ceiling of lg(range). */
|
||||
lg_range = ffs_u64(pow2_ceil_u64(range)) - 1;
|
||||
|
||||
/* Generate a result in [0..range) via repeated trial. */
|
||||
do {
|
||||
ret = prng_lg_range_u64(state, lg_range);
|
||||
} while (ret >= range);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE size_t
|
||||
prng_range_zu(size_t *state, size_t range, bool atomic)
|
||||
{
|
||||
size_t ret;
|
||||
unsigned lg_range;
|
||||
|
||||
assert(range > 1);
|
||||
|
||||
/* Compute the ceiling of lg(range). */
|
||||
lg_range = ffs_u64(pow2_ceil_u64(range)) - 1;
|
||||
|
||||
/* Generate a result in [0..range) via repeated trial. */
|
||||
do {
|
||||
ret = prng_lg_range_zu(state, lg_range, atomic);
|
||||
} while (ret >= range);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
|
||||
typedef struct prof_bt_s prof_bt_t;
|
||||
typedef struct prof_cnt_s prof_cnt_t;
|
||||
typedef struct prof_tctx_s prof_tctx_t;
|
||||
typedef struct prof_gctx_s prof_gctx_t;
|
||||
typedef struct prof_thr_cnt_s prof_thr_cnt_t;
|
||||
typedef struct prof_ctx_s prof_ctx_t;
|
||||
typedef struct prof_tdata_s prof_tdata_t;
|
||||
|
||||
/* Option defaults. */
|
||||
@@ -23,6 +23,9 @@ typedef struct prof_tdata_s prof_tdata_t;
|
||||
*/
|
||||
#define PROF_BT_MAX 128
|
||||
|
||||
/* Maximum number of backtraces to store in each per thread LRU cache. */
|
||||
#define PROF_TCMAX 1024
|
||||
|
||||
/* Initial hash table size. */
|
||||
#define PROF_CKH_MINITEMS 64
|
||||
|
||||
@@ -33,17 +36,11 @@ typedef struct prof_tdata_s prof_tdata_t;
|
||||
#define PROF_PRINTF_BUFSIZE 128
|
||||
|
||||
/*
|
||||
* Number of mutexes shared among all gctx's. No space is allocated for these
|
||||
* Number of mutexes shared among all ctx's. No space is allocated for these
|
||||
* unless profiling is enabled, so it's okay to over-provision.
|
||||
*/
|
||||
#define PROF_NCTX_LOCKS 1024
|
||||
|
||||
/*
|
||||
* Number of mutexes shared among all tdata's. No space is allocated for these
|
||||
* unless profiling is enabled, so it's okay to over-provision.
|
||||
*/
|
||||
#define PROF_NTDATA_LOCKS 256
|
||||
|
||||
/*
|
||||
* prof_tdata pointers close to NULL are used to encode state information that
|
||||
* is used for cleaning up during thread shutdown.
|
||||
@@ -66,186 +63,141 @@ struct prof_bt_s {
|
||||
/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */
|
||||
typedef struct {
|
||||
prof_bt_t *bt;
|
||||
unsigned nignore;
|
||||
unsigned max;
|
||||
} prof_unwind_data_t;
|
||||
#endif
|
||||
|
||||
struct prof_cnt_s {
|
||||
/* Profiling counters. */
|
||||
uint64_t curobjs;
|
||||
uint64_t curbytes;
|
||||
/*
|
||||
* Profiling counters. An allocation/deallocation pair can operate on
|
||||
* different prof_thr_cnt_t objects that are linked into the same
|
||||
* prof_ctx_t cnts_ql, so it is possible for the cur* counters to go
|
||||
* negative. In principle it is possible for the *bytes counters to
|
||||
* overflow/underflow, but a general solution would require something
|
||||
* like 128-bit counters; this implementation doesn't bother to solve
|
||||
* that problem.
|
||||
*/
|
||||
int64_t curobjs;
|
||||
int64_t curbytes;
|
||||
uint64_t accumobjs;
|
||||
uint64_t accumbytes;
|
||||
};
|
||||
|
||||
typedef enum {
|
||||
prof_tctx_state_initializing,
|
||||
prof_tctx_state_nominal,
|
||||
prof_tctx_state_dumping,
|
||||
prof_tctx_state_purgatory /* Dumper must finish destroying. */
|
||||
} prof_tctx_state_t;
|
||||
struct prof_thr_cnt_s {
|
||||
/* Linkage into prof_ctx_t's cnts_ql. */
|
||||
ql_elm(prof_thr_cnt_t) cnts_link;
|
||||
|
||||
struct prof_tctx_s {
|
||||
/* Thread data for thread that performed the allocation. */
|
||||
prof_tdata_t *tdata;
|
||||
/* Linkage into thread's LRU. */
|
||||
ql_elm(prof_thr_cnt_t) lru_link;
|
||||
|
||||
/*
|
||||
* Copy of tdata->thr_{uid,discrim}, necessary because tdata may be
|
||||
* defunct during teardown.
|
||||
* Associated context. If a thread frees an object that it did not
|
||||
* allocate, it is possible that the context is not cached in the
|
||||
* thread's hash table, in which case it must be able to look up the
|
||||
* context, insert a new prof_thr_cnt_t into the thread's hash table,
|
||||
* and link it into the prof_ctx_t's cnts_ql.
|
||||
*/
|
||||
uint64_t thr_uid;
|
||||
uint64_t thr_discrim;
|
||||
prof_ctx_t *ctx;
|
||||
|
||||
/* Profiling counters, protected by tdata->lock. */
|
||||
/*
|
||||
* Threads use memory barriers to update the counters. Since there is
|
||||
* only ever one writer, the only challenge is for the reader to get a
|
||||
* consistent read of the counters.
|
||||
*
|
||||
* The writer uses this series of operations:
|
||||
*
|
||||
* 1) Increment epoch to an odd number.
|
||||
* 2) Update counters.
|
||||
* 3) Increment epoch to an even number.
|
||||
*
|
||||
* The reader must assure 1) that the epoch is even while it reads the
|
||||
* counters, and 2) that the epoch doesn't change between the time it
|
||||
* starts and finishes reading the counters.
|
||||
*/
|
||||
unsigned epoch;
|
||||
|
||||
/* Profiling counters. */
|
||||
prof_cnt_t cnts;
|
||||
|
||||
/* Associated global context. */
|
||||
prof_gctx_t *gctx;
|
||||
|
||||
/*
|
||||
* UID that distinguishes multiple tctx's created by the same thread,
|
||||
* but coexisting in gctx->tctxs. There are two ways that such
|
||||
* coexistence can occur:
|
||||
* - A dumper thread can cause a tctx to be retained in the purgatory
|
||||
* state.
|
||||
* - Although a single "producer" thread must create all tctx's which
|
||||
* share the same thr_uid, multiple "consumers" can each concurrently
|
||||
* execute portions of prof_tctx_destroy(). prof_tctx_destroy() only
|
||||
* gets called once each time cnts.cur{objs,bytes} drop to 0, but this
|
||||
* threshold can be hit again before the first consumer finishes
|
||||
* executing prof_tctx_destroy().
|
||||
*/
|
||||
uint64_t tctx_uid;
|
||||
|
||||
/* Linkage into gctx's tctxs. */
|
||||
rb_node(prof_tctx_t) tctx_link;
|
||||
|
||||
/*
|
||||
* True during prof_alloc_prep()..prof_malloc_sample_object(), prevents
|
||||
* sample vs destroy race.
|
||||
*/
|
||||
bool prepared;
|
||||
|
||||
/* Current dump-related state, protected by gctx->lock. */
|
||||
prof_tctx_state_t state;
|
||||
|
||||
/*
|
||||
* Copy of cnts snapshotted during early dump phase, protected by
|
||||
* dump_mtx.
|
||||
*/
|
||||
prof_cnt_t dump_cnts;
|
||||
};
|
||||
typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;
|
||||
|
||||
struct prof_gctx_s {
|
||||
/* Protects nlimbo, cnt_summed, and tctxs. */
|
||||
struct prof_ctx_s {
|
||||
/* Associated backtrace. */
|
||||
prof_bt_t *bt;
|
||||
|
||||
/* Protects nlimbo, cnt_merged, and cnts_ql. */
|
||||
malloc_mutex_t *lock;
|
||||
|
||||
/*
|
||||
* Number of threads that currently cause this gctx to be in a state of
|
||||
* Number of threads that currently cause this ctx to be in a state of
|
||||
* limbo due to one of:
|
||||
* - Initializing this gctx.
|
||||
* - Initializing per thread counters associated with this gctx.
|
||||
* - Preparing to destroy this gctx.
|
||||
* - Dumping a heap profile that includes this gctx.
|
||||
* - Initializing per thread counters associated with this ctx.
|
||||
* - Preparing to destroy this ctx.
|
||||
* - Dumping a heap profile that includes this ctx.
|
||||
* nlimbo must be 1 (single destroyer) in order to safely destroy the
|
||||
* gctx.
|
||||
* ctx.
|
||||
*/
|
||||
unsigned nlimbo;
|
||||
|
||||
/*
|
||||
* Tree of profile counters, one for each thread that has allocated in
|
||||
* this context.
|
||||
*/
|
||||
prof_tctx_tree_t tctxs;
|
||||
|
||||
/* Linkage for tree of contexts to be dumped. */
|
||||
rb_node(prof_gctx_t) dump_link;
|
||||
|
||||
/* Temporary storage for summation during dump. */
|
||||
prof_cnt_t cnt_summed;
|
||||
|
||||
/* Associated backtrace. */
|
||||
prof_bt_t bt;
|
||||
/* When threads exit, they merge their stats into cnt_merged. */
|
||||
prof_cnt_t cnt_merged;
|
||||
|
||||
/* Backtrace vector, variable size, referred to by bt. */
|
||||
void *vec[1];
|
||||
/*
|
||||
* List of profile counters, one for each thread that has allocated in
|
||||
* this context.
|
||||
*/
|
||||
ql_head(prof_thr_cnt_t) cnts_ql;
|
||||
|
||||
/* Linkage for list of contexts to be dumped. */
|
||||
ql_elm(prof_ctx_t) dump_link;
|
||||
};
|
||||
typedef rb_tree(prof_gctx_t) prof_gctx_tree_t;
|
||||
typedef ql_head(prof_ctx_t) prof_ctx_list_t;
|
||||
|
||||
struct prof_tdata_s {
|
||||
malloc_mutex_t *lock;
|
||||
|
||||
/* Monotonically increasing unique thread identifier. */
|
||||
uint64_t thr_uid;
|
||||
|
||||
/*
|
||||
* Monotonically increasing discriminator among tdata structures
|
||||
* associated with the same thr_uid.
|
||||
* Hash of (prof_bt_t *)-->(prof_thr_cnt_t *). Each thread keeps a
|
||||
* cache of backtraces, with associated thread-specific prof_thr_cnt_t
|
||||
* objects. Other threads may read the prof_thr_cnt_t contents, but no
|
||||
* others will ever write them.
|
||||
*
|
||||
* Upon thread exit, the thread must merge all the prof_thr_cnt_t
|
||||
* counter data into the associated prof_ctx_t objects, and unlink/free
|
||||
* the prof_thr_cnt_t objects.
|
||||
*/
|
||||
uint64_t thr_discrim;
|
||||
ckh_t bt2cnt;
|
||||
|
||||
/* Included in heap profile dumps if non-NULL. */
|
||||
char *thread_name;
|
||||
/* LRU for contents of bt2cnt. */
|
||||
ql_head(prof_thr_cnt_t) lru_ql;
|
||||
|
||||
bool attached;
|
||||
bool expired;
|
||||
|
||||
rb_node(prof_tdata_t) tdata_link;
|
||||
|
||||
/*
|
||||
* Counter used to initialize prof_tctx_t's tctx_uid. No locking is
|
||||
* necessary when incrementing this field, because only one thread ever
|
||||
* does so.
|
||||
*/
|
||||
uint64_t tctx_uid_next;
|
||||
|
||||
/*
|
||||
* Hash of (prof_bt_t *)-->(prof_tctx_t *). Each thread tracks
|
||||
* backtraces for which it has non-zero allocation/deallocation counters
|
||||
* associated with thread-specific prof_tctx_t objects. Other threads
|
||||
* may write to prof_tctx_t contents when freeing associated objects.
|
||||
*/
|
||||
ckh_t bt2tctx;
|
||||
/* Backtrace vector, used for calls to prof_backtrace(). */
|
||||
void **vec;
|
||||
|
||||
/* Sampling state. */
|
||||
uint64_t prng_state;
|
||||
uint64_t bytes_until_sample;
|
||||
uint64_t threshold;
|
||||
uint64_t accum;
|
||||
|
||||
/* State used to avoid dumping while operating on prof internals. */
|
||||
bool enq;
|
||||
bool enq_idump;
|
||||
bool enq_gdump;
|
||||
|
||||
/*
|
||||
* Set to true during an early dump phase for tdata's which are
|
||||
* currently being dumped. New threads' tdata's have this initialized
|
||||
* to false so that they aren't accidentally included in later dump
|
||||
* phases.
|
||||
*/
|
||||
bool dumping;
|
||||
|
||||
/*
|
||||
* True if profiling is active for this tdata's thread
|
||||
* (thread.prof.active mallctl).
|
||||
*/
|
||||
bool active;
|
||||
|
||||
/* Temporary storage for summation during dump. */
|
||||
prof_cnt_t cnt_summed;
|
||||
|
||||
/* Backtrace vector, used for calls to prof_backtrace(). */
|
||||
void *vec[PROF_BT_MAX];
|
||||
};
|
||||
typedef rb_tree(prof_tdata_t) prof_tdata_tree_t;
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
extern bool opt_prof;
|
||||
/*
|
||||
* Even if opt_prof is true, sampling can be temporarily disabled by setting
|
||||
* opt_prof_active to false. No locking is used when updating opt_prof_active,
|
||||
* so there are no guarantees regarding how long it will take for all threads
|
||||
* to notice state changes.
|
||||
*/
|
||||
extern bool opt_prof_active;
|
||||
extern bool opt_prof_thread_active_init;
|
||||
extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */
|
||||
extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */
|
||||
extern bool opt_prof_gdump; /* High-water memory dumping. */
|
||||
@@ -259,12 +211,6 @@ extern char opt_prof_prefix[
|
||||
#endif
|
||||
1];
|
||||
|
||||
/* Accessed via prof_active_[gs]et{_unlocked,}(). */
|
||||
extern bool prof_active;
|
||||
|
||||
/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */
|
||||
extern bool prof_gdump_val;
|
||||
|
||||
/*
|
||||
* Profile dump interval, measured in bytes allocated. Each arena triggers a
|
||||
* profile dump when it reaches this threshold. The effect is that the
|
||||
@@ -275,271 +221,391 @@ extern bool prof_gdump_val;
|
||||
extern uint64_t prof_interval;
|
||||
|
||||
/*
|
||||
* Initialized as opt_lg_prof_sample, and potentially modified during profiling
|
||||
* resets.
|
||||
* If true, promote small sampled objects to large objects, since small run
|
||||
* headers do not have embedded profile context pointers.
|
||||
*/
|
||||
extern size_t lg_prof_sample;
|
||||
extern bool prof_promote;
|
||||
|
||||
void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated);
|
||||
void prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
|
||||
prof_tctx_t *tctx);
|
||||
void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx);
|
||||
void bt_init(prof_bt_t *bt, void **vec);
|
||||
void prof_backtrace(prof_bt_t *bt);
|
||||
prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt);
|
||||
void prof_backtrace(prof_bt_t *bt, unsigned nignore);
|
||||
prof_thr_cnt_t *prof_lookup(prof_bt_t *bt);
|
||||
#ifdef JEMALLOC_JET
|
||||
size_t prof_tdata_count(void);
|
||||
size_t prof_bt_count(void);
|
||||
const prof_cnt_t *prof_cnt_all(void);
|
||||
typedef int (prof_dump_open_t)(bool, const char *);
|
||||
extern prof_dump_open_t *prof_dump_open;
|
||||
typedef bool (prof_dump_header_t)(tsdn_t *, bool, const prof_cnt_t *);
|
||||
extern prof_dump_header_t *prof_dump_header;
|
||||
#endif
|
||||
void prof_idump(tsdn_t *tsdn);
|
||||
bool prof_mdump(tsd_t *tsd, const char *filename);
|
||||
void prof_gdump(tsdn_t *tsdn);
|
||||
prof_tdata_t *prof_tdata_init(tsd_t *tsd);
|
||||
prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
|
||||
void prof_reset(tsd_t *tsd, size_t lg_sample);
|
||||
void prof_tdata_cleanup(tsd_t *tsd);
|
||||
bool prof_active_get(tsdn_t *tsdn);
|
||||
bool prof_active_set(tsdn_t *tsdn, bool active);
|
||||
const char *prof_thread_name_get(tsd_t *tsd);
|
||||
int prof_thread_name_set(tsd_t *tsd, const char *thread_name);
|
||||
bool prof_thread_active_get(tsd_t *tsd);
|
||||
bool prof_thread_active_set(tsd_t *tsd, bool active);
|
||||
bool prof_thread_active_init_get(tsdn_t *tsdn);
|
||||
bool prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
|
||||
bool prof_gdump_get(tsdn_t *tsdn);
|
||||
bool prof_gdump_set(tsdn_t *tsdn, bool active);
|
||||
void prof_idump(void);
|
||||
bool prof_mdump(const char *filename);
|
||||
void prof_gdump(void);
|
||||
prof_tdata_t *prof_tdata_init(void);
|
||||
void prof_tdata_cleanup(void *arg);
|
||||
void prof_boot0(void);
|
||||
void prof_boot1(void);
|
||||
bool prof_boot2(tsd_t *tsd);
|
||||
void prof_prefork0(tsdn_t *tsdn);
|
||||
void prof_prefork1(tsdn_t *tsdn);
|
||||
void prof_postfork_parent(tsdn_t *tsdn);
|
||||
void prof_postfork_child(tsdn_t *tsdn);
|
||||
void prof_sample_threshold_update(prof_tdata_t *tdata);
|
||||
bool prof_boot2(void);
|
||||
void prof_prefork(void);
|
||||
void prof_postfork_parent(void);
|
||||
void prof_postfork_child(void);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#define PROF_ALLOC_PREP(nignore, size, ret) do { \
|
||||
prof_tdata_t *prof_tdata; \
|
||||
prof_bt_t bt; \
|
||||
\
|
||||
assert(size == s2u(size)); \
|
||||
\
|
||||
prof_tdata = prof_tdata_get(true); \
|
||||
if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) { \
|
||||
if (prof_tdata != NULL) \
|
||||
ret = (prof_thr_cnt_t *)(uintptr_t)1U; \
|
||||
else \
|
||||
ret = NULL; \
|
||||
break; \
|
||||
} \
|
||||
\
|
||||
if (opt_prof_active == false) { \
|
||||
/* Sampling is currently inactive, so avoid sampling. */\
|
||||
ret = (prof_thr_cnt_t *)(uintptr_t)1U; \
|
||||
} else if (opt_lg_prof_sample == 0) { \
|
||||
/* Don't bother with sampling logic, since sampling */\
|
||||
/* interval is 1. */\
|
||||
bt_init(&bt, prof_tdata->vec); \
|
||||
prof_backtrace(&bt, nignore); \
|
||||
ret = prof_lookup(&bt); \
|
||||
} else { \
|
||||
if (prof_tdata->threshold == 0) { \
|
||||
/* Initialize. Seed the prng differently for */\
|
||||
/* each thread. */\
|
||||
prof_tdata->prng_state = \
|
||||
(uint64_t)(uintptr_t)&size; \
|
||||
prof_sample_threshold_update(prof_tdata); \
|
||||
} \
|
||||
\
|
||||
/* Determine whether to capture a backtrace based on */\
|
||||
/* whether size is enough for prof_accum to reach */\
|
||||
/* prof_tdata->threshold. However, delay updating */\
|
||||
/* these variables until prof_{m,re}alloc(), because */\
|
||||
/* we don't know for sure that the allocation will */\
|
||||
/* succeed. */\
|
||||
/* */\
|
||||
/* Use subtraction rather than addition to avoid */\
|
||||
/* potential integer overflow. */\
|
||||
if (size >= prof_tdata->threshold - \
|
||||
prof_tdata->accum) { \
|
||||
bt_init(&bt, prof_tdata->vec); \
|
||||
prof_backtrace(&bt, nignore); \
|
||||
ret = prof_lookup(&bt); \
|
||||
} else \
|
||||
ret = (prof_thr_cnt_t *)(uintptr_t)1U; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
bool prof_active_get_unlocked(void);
|
||||
bool prof_gdump_get_unlocked(void);
|
||||
prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create);
|
||||
prof_tctx_t *prof_tctx_get(tsdn_t *tsdn, const void *ptr);
|
||||
void prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
|
||||
prof_tctx_t *tctx);
|
||||
void prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize,
|
||||
const void *old_ptr, prof_tctx_t *tctx);
|
||||
bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit,
|
||||
prof_tdata_t **tdata_out);
|
||||
prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active,
|
||||
bool update);
|
||||
void prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize,
|
||||
prof_tctx_t *tctx);
|
||||
void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize,
|
||||
prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr,
|
||||
size_t old_usize, prof_tctx_t *old_tctx);
|
||||
void prof_free(tsd_t *tsd, const void *ptr, size_t usize);
|
||||
malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *)
|
||||
|
||||
prof_tdata_t *prof_tdata_get(bool create);
|
||||
void prof_sample_threshold_update(prof_tdata_t *prof_tdata);
|
||||
prof_ctx_t *prof_ctx_get(const void *ptr);
|
||||
void prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx);
|
||||
bool prof_sample_accum_update(size_t size);
|
||||
void prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt);
|
||||
void prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt,
|
||||
size_t old_usize, prof_ctx_t *old_ctx);
|
||||
void prof_free(const void *ptr, size_t size);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
prof_active_get_unlocked(void)
|
||||
/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */
|
||||
malloc_tsd_externs(prof_tdata, prof_tdata_t *)
|
||||
malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL,
|
||||
prof_tdata_cleanup)
|
||||
|
||||
JEMALLOC_INLINE prof_tdata_t *
|
||||
prof_tdata_get(bool create)
|
||||
{
|
||||
prof_tdata_t *prof_tdata;
|
||||
|
||||
cassert(config_prof);
|
||||
|
||||
prof_tdata = *prof_tdata_tsd_get();
|
||||
if (create && prof_tdata == NULL)
|
||||
prof_tdata = prof_tdata_init();
|
||||
|
||||
return (prof_tdata);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
prof_sample_threshold_update(prof_tdata_t *prof_tdata)
|
||||
{
|
||||
/*
|
||||
* The body of this function is compiled out unless heap profiling is
|
||||
* enabled, so that it is possible to compile jemalloc with floating
|
||||
* point support completely disabled. Avoiding floating point code is
|
||||
* important on memory-constrained systems, but it also enables a
|
||||
* workaround for versions of glibc that don't properly save/restore
|
||||
* floating point registers during dynamic lazy symbol loading (which
|
||||
* internally calls into whatever malloc implementation happens to be
|
||||
* integrated into the application). Note that some compilers (e.g.
|
||||
* gcc 4.8) may use floating point registers for fast memory moves, so
|
||||
* jemalloc must be compiled with such optimizations disabled (e.g.
|
||||
* -mno-sse) in order for the workaround to be complete.
|
||||
*/
|
||||
#ifdef JEMALLOC_PROF
|
||||
uint64_t r;
|
||||
double u;
|
||||
|
||||
cassert(config_prof);
|
||||
|
||||
/*
|
||||
* Even if opt_prof is true, sampling can be temporarily disabled by
|
||||
* setting prof_active to false. No locking is used when reading
|
||||
* prof_active in the fast path, so there are no guarantees regarding
|
||||
* how long it will take for all threads to notice state changes.
|
||||
* Compute sample threshold as a geometrically distributed random
|
||||
* variable with mean (2^opt_lg_prof_sample).
|
||||
*
|
||||
* __ __
|
||||
* | log(u) | 1
|
||||
* prof_tdata->threshold = | -------- |, where p = -------------------
|
||||
* | log(1-p) | opt_lg_prof_sample
|
||||
* 2
|
||||
*
|
||||
* For more information on the math, see:
|
||||
*
|
||||
* Non-Uniform Random Variate Generation
|
||||
* Luc Devroye
|
||||
* Springer-Verlag, New York, 1986
|
||||
* pp 500
|
||||
* (http://luc.devroye.org/rnbookindex.html)
|
||||
*/
|
||||
return (prof_active);
|
||||
prng64(r, 53, prof_tdata->prng_state,
|
||||
UINT64_C(6364136223846793005), UINT64_C(1442695040888963407));
|
||||
u = (double)r * (1.0/9007199254740992.0L);
|
||||
prof_tdata->threshold = (uint64_t)(log(u) /
|
||||
log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample))))
|
||||
+ (uint64_t)1U;
|
||||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
prof_gdump_get_unlocked(void)
|
||||
{
|
||||
|
||||
/*
|
||||
* No locking is used when reading prof_gdump_val in the fast path, so
|
||||
* there are no guarantees regarding how long it will take for all
|
||||
* threads to notice state changes.
|
||||
*/
|
||||
return (prof_gdump_val);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE prof_tdata_t *
|
||||
prof_tdata_get(tsd_t *tsd, bool create)
|
||||
{
|
||||
prof_tdata_t *tdata;
|
||||
|
||||
cassert(config_prof);
|
||||
|
||||
tdata = tsd_prof_tdata_get(tsd);
|
||||
if (create) {
|
||||
if (unlikely(tdata == NULL)) {
|
||||
if (tsd_nominal(tsd)) {
|
||||
tdata = prof_tdata_init(tsd);
|
||||
tsd_prof_tdata_set(tsd, tdata);
|
||||
}
|
||||
} else if (unlikely(tdata->expired)) {
|
||||
tdata = prof_tdata_reinit(tsd, tdata);
|
||||
tsd_prof_tdata_set(tsd, tdata);
|
||||
}
|
||||
assert(tdata == NULL || tdata->attached);
|
||||
}
|
||||
|
||||
return (tdata);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE prof_tctx_t *
|
||||
prof_tctx_get(tsdn_t *tsdn, const void *ptr)
|
||||
JEMALLOC_INLINE prof_ctx_t *
|
||||
prof_ctx_get(const void *ptr)
|
||||
{
|
||||
prof_ctx_t *ret;
|
||||
arena_chunk_t *chunk;
|
||||
|
||||
cassert(config_prof);
|
||||
assert(ptr != NULL);
|
||||
|
||||
return (arena_prof_tctx_get(tsdn, ptr));
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx)
|
||||
{
|
||||
|
||||
cassert(config_prof);
|
||||
assert(ptr != NULL);
|
||||
|
||||
arena_prof_tctx_set(tsdn, ptr, usize, tctx);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, const void *old_ptr,
|
||||
prof_tctx_t *old_tctx)
|
||||
{
|
||||
|
||||
cassert(config_prof);
|
||||
assert(ptr != NULL);
|
||||
|
||||
arena_prof_tctx_reset(tsdn, ptr, usize, old_ptr, old_tctx);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
|
||||
prof_tdata_t **tdata_out)
|
||||
{
|
||||
prof_tdata_t *tdata;
|
||||
|
||||
cassert(config_prof);
|
||||
|
||||
tdata = prof_tdata_get(tsd, true);
|
||||
if (unlikely((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX))
|
||||
tdata = NULL;
|
||||
|
||||
if (tdata_out != NULL)
|
||||
*tdata_out = tdata;
|
||||
|
||||
if (unlikely(tdata == NULL))
|
||||
return (true);
|
||||
|
||||
if (likely(tdata->bytes_until_sample >= usize)) {
|
||||
if (update)
|
||||
tdata->bytes_until_sample -= usize;
|
||||
return (true);
|
||||
} else {
|
||||
/* Compute new sample threshold. */
|
||||
if (update)
|
||||
prof_sample_threshold_update(tdata);
|
||||
return (!tdata->active);
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE prof_tctx_t *
|
||||
prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update)
|
||||
{
|
||||
prof_tctx_t *ret;
|
||||
prof_tdata_t *tdata;
|
||||
prof_bt_t bt;
|
||||
|
||||
assert(usize == s2u(usize));
|
||||
|
||||
if (!prof_active || likely(prof_sample_accum_update(tsd, usize, update,
|
||||
&tdata)))
|
||||
ret = (prof_tctx_t *)(uintptr_t)1U;
|
||||
else {
|
||||
bt_init(&bt, tdata->vec);
|
||||
prof_backtrace(&bt);
|
||||
ret = prof_lookup(tsd, &bt);
|
||||
}
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||
if (chunk != ptr) {
|
||||
/* Region. */
|
||||
ret = arena_prof_ctx_get(ptr);
|
||||
} else
|
||||
ret = huge_prof_ctx_get(ptr);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx)
|
||||
JEMALLOC_INLINE void
|
||||
prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx)
|
||||
{
|
||||
arena_chunk_t *chunk;
|
||||
|
||||
cassert(config_prof);
|
||||
assert(ptr != NULL);
|
||||
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||
if (chunk != ptr) {
|
||||
/* Region. */
|
||||
arena_prof_ctx_set(ptr, usize, ctx);
|
||||
} else
|
||||
huge_prof_ctx_set(ptr, ctx);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
prof_sample_accum_update(size_t size)
|
||||
{
|
||||
prof_tdata_t *prof_tdata;
|
||||
|
||||
cassert(config_prof);
|
||||
/* Sampling logic is unnecessary if the interval is 1. */
|
||||
assert(opt_lg_prof_sample != 0);
|
||||
|
||||
prof_tdata = prof_tdata_get(false);
|
||||
if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
|
||||
return (true);
|
||||
|
||||
/* Take care to avoid integer overflow. */
|
||||
if (size >= prof_tdata->threshold - prof_tdata->accum) {
|
||||
prof_tdata->accum -= (prof_tdata->threshold - size);
|
||||
/* Compute new sample threshold. */
|
||||
prof_sample_threshold_update(prof_tdata);
|
||||
while (prof_tdata->accum >= prof_tdata->threshold) {
|
||||
prof_tdata->accum -= prof_tdata->threshold;
|
||||
prof_sample_threshold_update(prof_tdata);
|
||||
}
|
||||
return (false);
|
||||
} else {
|
||||
prof_tdata->accum += size;
|
||||
return (true);
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt)
|
||||
{
|
||||
|
||||
cassert(config_prof);
|
||||
assert(ptr != NULL);
|
||||
assert(usize == isalloc(tsdn, ptr, true));
|
||||
assert(usize == isalloc(ptr, true));
|
||||
|
||||
if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
|
||||
prof_malloc_sample_object(tsdn, ptr, usize, tctx);
|
||||
else
|
||||
prof_tctx_set(tsdn, ptr, usize, (prof_tctx_t *)(uintptr_t)1U);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
|
||||
bool prof_active, bool updated, const void *old_ptr, size_t old_usize,
|
||||
prof_tctx_t *old_tctx)
|
||||
{
|
||||
bool sampled, old_sampled;
|
||||
|
||||
cassert(config_prof);
|
||||
assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U);
|
||||
|
||||
if (prof_active && !updated && ptr != NULL) {
|
||||
assert(usize == isalloc(tsd_tsdn(tsd), ptr, true));
|
||||
if (prof_sample_accum_update(tsd, usize, true, NULL)) {
|
||||
if (opt_lg_prof_sample != 0) {
|
||||
if (prof_sample_accum_update(usize)) {
|
||||
/*
|
||||
* Don't sample. The usize passed to prof_alloc_prep()
|
||||
* was larger than what actually got allocated, so a
|
||||
* backtrace was captured for this allocation, even
|
||||
* though its actual usize was insufficient to cross the
|
||||
* sample threshold.
|
||||
* Don't sample. For malloc()-like allocation, it is
|
||||
* always possible to tell in advance how large an
|
||||
* object's usable size will be, so there should never
|
||||
* be a difference between the usize passed to
|
||||
* PROF_ALLOC_PREP() and prof_malloc().
|
||||
*/
|
||||
prof_alloc_rollback(tsd, tctx, true);
|
||||
tctx = (prof_tctx_t *)(uintptr_t)1U;
|
||||
assert((uintptr_t)cnt == (uintptr_t)1U);
|
||||
}
|
||||
}
|
||||
|
||||
sampled = ((uintptr_t)tctx > (uintptr_t)1U);
|
||||
old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U);
|
||||
if ((uintptr_t)cnt > (uintptr_t)1U) {
|
||||
prof_ctx_set(ptr, usize, cnt->ctx);
|
||||
|
||||
if (unlikely(sampled))
|
||||
prof_malloc_sample_object(tsd_tsdn(tsd), ptr, usize, tctx);
|
||||
else
|
||||
prof_tctx_reset(tsd_tsdn(tsd), ptr, usize, old_ptr, old_tctx);
|
||||
|
||||
if (unlikely(old_sampled))
|
||||
prof_free_sampled_object(tsd, old_usize, old_tctx);
|
||||
cnt->epoch++;
|
||||
/*********/
|
||||
mb_write();
|
||||
/*********/
|
||||
cnt->cnts.curobjs++;
|
||||
cnt->cnts.curbytes += usize;
|
||||
if (opt_prof_accum) {
|
||||
cnt->cnts.accumobjs++;
|
||||
cnt->cnts.accumbytes += usize;
|
||||
}
|
||||
/*********/
|
||||
mb_write();
|
||||
/*********/
|
||||
cnt->epoch++;
|
||||
/*********/
|
||||
mb_write();
|
||||
/*********/
|
||||
} else
|
||||
prof_ctx_set(ptr, usize, (prof_ctx_t *)(uintptr_t)1U);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
prof_free(tsd_t *tsd, const void *ptr, size_t usize)
|
||||
JEMALLOC_INLINE void
|
||||
prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt,
|
||||
size_t old_usize, prof_ctx_t *old_ctx)
|
||||
{
|
||||
prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), ptr);
|
||||
prof_thr_cnt_t *told_cnt;
|
||||
|
||||
cassert(config_prof);
|
||||
assert(usize == isalloc(tsd_tsdn(tsd), ptr, true));
|
||||
assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U);
|
||||
|
||||
if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
|
||||
prof_free_sampled_object(tsd, usize, tctx);
|
||||
if (ptr != NULL) {
|
||||
assert(usize == isalloc(ptr, true));
|
||||
if (opt_lg_prof_sample != 0) {
|
||||
if (prof_sample_accum_update(usize)) {
|
||||
/*
|
||||
* Don't sample. The usize passed to
|
||||
* PROF_ALLOC_PREP() was larger than what
|
||||
* actually got allocated, so a backtrace was
|
||||
* captured for this allocation, even though
|
||||
* its actual usize was insufficient to cross
|
||||
* the sample threshold.
|
||||
*/
|
||||
cnt = (prof_thr_cnt_t *)(uintptr_t)1U;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((uintptr_t)old_ctx > (uintptr_t)1U) {
|
||||
told_cnt = prof_lookup(old_ctx->bt);
|
||||
if (told_cnt == NULL) {
|
||||
/*
|
||||
* It's too late to propagate OOM for this realloc(),
|
||||
* so operate directly on old_cnt->ctx->cnt_merged.
|
||||
*/
|
||||
malloc_mutex_lock(old_ctx->lock);
|
||||
old_ctx->cnt_merged.curobjs--;
|
||||
old_ctx->cnt_merged.curbytes -= old_usize;
|
||||
malloc_mutex_unlock(old_ctx->lock);
|
||||
told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U;
|
||||
}
|
||||
} else
|
||||
told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U;
|
||||
|
||||
if ((uintptr_t)told_cnt > (uintptr_t)1U)
|
||||
told_cnt->epoch++;
|
||||
if ((uintptr_t)cnt > (uintptr_t)1U) {
|
||||
prof_ctx_set(ptr, usize, cnt->ctx);
|
||||
cnt->epoch++;
|
||||
} else if (ptr != NULL)
|
||||
prof_ctx_set(ptr, usize, (prof_ctx_t *)(uintptr_t)1U);
|
||||
/*********/
|
||||
mb_write();
|
||||
/*********/
|
||||
if ((uintptr_t)told_cnt > (uintptr_t)1U) {
|
||||
told_cnt->cnts.curobjs--;
|
||||
told_cnt->cnts.curbytes -= old_usize;
|
||||
}
|
||||
if ((uintptr_t)cnt > (uintptr_t)1U) {
|
||||
cnt->cnts.curobjs++;
|
||||
cnt->cnts.curbytes += usize;
|
||||
if (opt_prof_accum) {
|
||||
cnt->cnts.accumobjs++;
|
||||
cnt->cnts.accumbytes += usize;
|
||||
}
|
||||
}
|
||||
/*********/
|
||||
mb_write();
|
||||
/*********/
|
||||
if ((uintptr_t)told_cnt > (uintptr_t)1U)
|
||||
told_cnt->epoch++;
|
||||
if ((uintptr_t)cnt > (uintptr_t)1U)
|
||||
cnt->epoch++;
|
||||
/*********/
|
||||
mb_write(); /* Not strictly necessary. */
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
prof_free(const void *ptr, size_t size)
|
||||
{
|
||||
prof_ctx_t *ctx = prof_ctx_get(ptr);
|
||||
|
||||
cassert(config_prof);
|
||||
|
||||
if ((uintptr_t)ctx > (uintptr_t)1) {
|
||||
prof_thr_cnt_t *tcnt;
|
||||
assert(size == isalloc(ptr, true));
|
||||
tcnt = prof_lookup(ctx->bt);
|
||||
|
||||
if (tcnt != NULL) {
|
||||
tcnt->epoch++;
|
||||
/*********/
|
||||
mb_write();
|
||||
/*********/
|
||||
tcnt->cnts.curobjs--;
|
||||
tcnt->cnts.curbytes -= size;
|
||||
/*********/
|
||||
mb_write();
|
||||
/*********/
|
||||
tcnt->epoch++;
|
||||
/*********/
|
||||
mb_write();
|
||||
/*********/
|
||||
} else {
|
||||
/*
|
||||
* OOM during free() cannot be propagated, so operate
|
||||
* directly on cnt->ctx->cnt_merged.
|
||||
*/
|
||||
malloc_mutex_lock(ctx->lock);
|
||||
ctx->cnt_merged.curobjs--;
|
||||
ctx->cnt_merged.curbytes -= size;
|
||||
malloc_mutex_unlock(ctx->lock);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
/* List definitions. */
|
||||
/*
|
||||
* List definitions.
|
||||
*/
|
||||
#define ql_head(a_type) \
|
||||
struct { \
|
||||
a_type *qlh_first; \
|
||||
|
||||
@@ -40,10 +40,8 @@ struct { \
|
||||
(a_qr_b)->a_field.qre_prev = t; \
|
||||
} while (0)
|
||||
|
||||
/*
|
||||
* qr_meld() and qr_split() are functionally equivalent, so there's no need to
|
||||
* have two copies of the code.
|
||||
*/
|
||||
/* qr_meld() and qr_split() are functionally equivalent, so there's no need to
|
||||
* have two copies of the code. */
|
||||
#define qr_split(a_qr_a, a_qr_b, a_field) \
|
||||
qr_meld((a_qr_a), (a_qr_b), a_field)
|
||||
|
||||
|
||||
@@ -29,29 +29,36 @@ struct quarantine_s {
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
void quarantine_alloc_hook_work(tsd_t *tsd);
|
||||
void quarantine(tsd_t *tsd, void *ptr);
|
||||
void quarantine_cleanup(tsd_t *tsd);
|
||||
quarantine_t *quarantine_init(size_t lg_maxobjs);
|
||||
void quarantine(void *ptr);
|
||||
void quarantine_cleanup(void *arg);
|
||||
bool quarantine_boot(void);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
malloc_tsd_protos(JEMALLOC_ATTR(unused), quarantine, quarantine_t *)
|
||||
|
||||
void quarantine_alloc_hook(void);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_QUARANTINE_C_))
|
||||
malloc_tsd_externs(quarantine, quarantine_t *)
|
||||
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, quarantine, quarantine_t *, NULL,
|
||||
quarantine_cleanup)
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
quarantine_alloc_hook(void)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
quarantine_t *quarantine;
|
||||
|
||||
assert(config_fill && opt_quarantine);
|
||||
|
||||
tsd = tsd_fetch();
|
||||
if (tsd_quarantine_get(tsd) == NULL)
|
||||
quarantine_alloc_hook_work(tsd);
|
||||
quarantine = *quarantine_tsd_get();
|
||||
if (quarantine == NULL)
|
||||
quarantine_init(LG_MAXOBJS_INIT);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ struct { \
|
||||
#define rb_tree(a_type) \
|
||||
struct { \
|
||||
a_type *rbt_root; \
|
||||
a_type rbt_nil; \
|
||||
}
|
||||
|
||||
/* Left accessors. */
|
||||
@@ -78,15 +79,6 @@ struct { \
|
||||
(a_node)->a_field.rbn_right_red = (a_type *) (((intptr_t) \
|
||||
(a_node)->a_field.rbn_right_red) & ((ssize_t)-2)); \
|
||||
} while (0)
|
||||
|
||||
/* Node initializer. */
|
||||
#define rbt_node_new(a_type, a_field, a_rbt, a_node) do { \
|
||||
/* Bookkeeping bit cannot be used by node pointer. */ \
|
||||
assert(((uintptr_t)(a_node) & 0x1) == 0); \
|
||||
rbtn_left_set(a_type, a_field, (a_node), NULL); \
|
||||
rbtn_right_set(a_type, a_field, (a_node), NULL); \
|
||||
rbtn_red_set(a_type, a_field, (a_node)); \
|
||||
} while (0)
|
||||
#else
|
||||
/* Right accessors. */
|
||||
#define rbtn_right_get(a_type, a_field, a_node) \
|
||||
@@ -107,26 +99,28 @@ struct { \
|
||||
#define rbtn_black_set(a_type, a_field, a_node) do { \
|
||||
(a_node)->a_field.rbn_red = false; \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
/* Node initializer. */
|
||||
#define rbt_node_new(a_type, a_field, a_rbt, a_node) do { \
|
||||
rbtn_left_set(a_type, a_field, (a_node), NULL); \
|
||||
rbtn_right_set(a_type, a_field, (a_node), NULL); \
|
||||
rbtn_left_set(a_type, a_field, (a_node), &(a_rbt)->rbt_nil); \
|
||||
rbtn_right_set(a_type, a_field, (a_node), &(a_rbt)->rbt_nil); \
|
||||
rbtn_red_set(a_type, a_field, (a_node)); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
/* Tree initializer. */
|
||||
#define rb_new(a_type, a_field, a_rbt) do { \
|
||||
(a_rbt)->rbt_root = NULL; \
|
||||
(a_rbt)->rbt_root = &(a_rbt)->rbt_nil; \
|
||||
rbt_node_new(a_type, a_field, a_rbt, &(a_rbt)->rbt_nil); \
|
||||
rbtn_black_set(a_type, a_field, &(a_rbt)->rbt_nil); \
|
||||
} while (0)
|
||||
|
||||
/* Internal utility macros. */
|
||||
#define rbtn_first(a_type, a_field, a_rbt, a_root, r_node) do { \
|
||||
(r_node) = (a_root); \
|
||||
if ((r_node) != NULL) { \
|
||||
if ((r_node) != &(a_rbt)->rbt_nil) { \
|
||||
for (; \
|
||||
rbtn_left_get(a_type, a_field, (r_node)) != NULL; \
|
||||
rbtn_left_get(a_type, a_field, (r_node)) != &(a_rbt)->rbt_nil;\
|
||||
(r_node) = rbtn_left_get(a_type, a_field, (r_node))) { \
|
||||
} \
|
||||
} \
|
||||
@@ -134,9 +128,10 @@ struct { \
|
||||
|
||||
#define rbtn_last(a_type, a_field, a_rbt, a_root, r_node) do { \
|
||||
(r_node) = (a_root); \
|
||||
if ((r_node) != NULL) { \
|
||||
for (; rbtn_right_get(a_type, a_field, (r_node)) != NULL; \
|
||||
(r_node) = rbtn_right_get(a_type, a_field, (r_node))) { \
|
||||
if ((r_node) != &(a_rbt)->rbt_nil) { \
|
||||
for (; rbtn_right_get(a_type, a_field, (r_node)) != \
|
||||
&(a_rbt)->rbt_nil; (r_node) = rbtn_right_get(a_type, a_field, \
|
||||
(r_node))) { \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
@@ -163,8 +158,6 @@ struct { \
|
||||
#define rb_proto(a_attr, a_prefix, a_rbt_type, a_type) \
|
||||
a_attr void \
|
||||
a_prefix##new(a_rbt_type *rbtree); \
|
||||
a_attr bool \
|
||||
a_prefix##empty(a_rbt_type *rbtree); \
|
||||
a_attr a_type * \
|
||||
a_prefix##first(a_rbt_type *rbtree); \
|
||||
a_attr a_type * \
|
||||
@@ -174,11 +167,11 @@ a_prefix##next(a_rbt_type *rbtree, a_type *node); \
|
||||
a_attr a_type * \
|
||||
a_prefix##prev(a_rbt_type *rbtree, a_type *node); \
|
||||
a_attr a_type * \
|
||||
a_prefix##search(a_rbt_type *rbtree, const a_type *key); \
|
||||
a_prefix##search(a_rbt_type *rbtree, a_type *key); \
|
||||
a_attr a_type * \
|
||||
a_prefix##nsearch(a_rbt_type *rbtree, const a_type *key); \
|
||||
a_prefix##nsearch(a_rbt_type *rbtree, a_type *key); \
|
||||
a_attr a_type * \
|
||||
a_prefix##psearch(a_rbt_type *rbtree, const a_type *key); \
|
||||
a_prefix##psearch(a_rbt_type *rbtree, a_type *key); \
|
||||
a_attr void \
|
||||
a_prefix##insert(a_rbt_type *rbtree, a_type *node); \
|
||||
a_attr void \
|
||||
@@ -188,10 +181,7 @@ a_prefix##iter(a_rbt_type *rbtree, a_type *start, a_type *(*cb)( \
|
||||
a_rbt_type *, a_type *, void *), void *arg); \
|
||||
a_attr a_type * \
|
||||
a_prefix##reverse_iter(a_rbt_type *rbtree, a_type *start, \
|
||||
a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg); \
|
||||
a_attr void \
|
||||
a_prefix##destroy(a_rbt_type *rbtree, void (*cb)(a_type *, void *), \
|
||||
void *arg);
|
||||
a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg);
|
||||
|
||||
/*
|
||||
* The rb_gen() macro generates a type-specific red-black tree implementation,
|
||||
@@ -208,7 +198,7 @@ a_prefix##destroy(a_rbt_type *rbtree, void (*cb)(a_type *, void *), \
|
||||
* int (a_cmp *)(a_type *a_node, a_type *a_other);
|
||||
* ^^^^^^
|
||||
* or a_key
|
||||
* Interpretation of comparison function return values:
|
||||
* Interpretation of comparision function return values:
|
||||
* -1 : a_node < a_other
|
||||
* 0 : a_node == a_other
|
||||
* 1 : a_node > a_other
|
||||
@@ -234,13 +224,6 @@ a_prefix##destroy(a_rbt_type *rbtree, void (*cb)(a_type *, void *), \
|
||||
* Args:
|
||||
* tree: Pointer to an uninitialized red-black tree object.
|
||||
*
|
||||
* static bool
|
||||
* ex_empty(ex_t *tree);
|
||||
* Description: Determine whether tree is empty.
|
||||
* Args:
|
||||
* tree: Pointer to an initialized red-black tree object.
|
||||
* Ret: True if tree is empty, false otherwise.
|
||||
*
|
||||
* static ex_node_t *
|
||||
* ex_first(ex_t *tree);
|
||||
* static ex_node_t *
|
||||
@@ -262,7 +245,7 @@ a_prefix##destroy(a_rbt_type *rbtree, void (*cb)(a_type *, void *), \
|
||||
* last/first.
|
||||
*
|
||||
* static ex_node_t *
|
||||
* ex_search(ex_t *tree, const ex_node_t *key);
|
||||
* ex_search(ex_t *tree, ex_node_t *key);
|
||||
* Description: Search for node that matches key.
|
||||
* Args:
|
||||
* tree: Pointer to an initialized red-black tree object.
|
||||
@@ -270,9 +253,9 @@ a_prefix##destroy(a_rbt_type *rbtree, void (*cb)(a_type *, void *), \
|
||||
* Ret: Node in tree that matches key, or NULL if no match.
|
||||
*
|
||||
* static ex_node_t *
|
||||
* ex_nsearch(ex_t *tree, const ex_node_t *key);
|
||||
* ex_nsearch(ex_t *tree, ex_node_t *key);
|
||||
* static ex_node_t *
|
||||
* ex_psearch(ex_t *tree, const ex_node_t *key);
|
||||
* ex_psearch(ex_t *tree, ex_node_t *key);
|
||||
* Description: Search for node that matches key. If no match is found,
|
||||
* return what would be key's successor/predecessor, were
|
||||
* key in tree.
|
||||
@@ -320,52 +303,40 @@ a_prefix##destroy(a_rbt_type *rbtree, void (*cb)(a_type *, void *), \
|
||||
* arg : Opaque pointer passed to cb().
|
||||
* Ret: NULL if iteration completed, or the non-NULL callback return value
|
||||
* that caused termination of the iteration.
|
||||
*
|
||||
* static void
|
||||
* ex_destroy(ex_t *tree, void (*cb)(ex_node_t *, void *), void *arg);
|
||||
* Description: Iterate over the tree with post-order traversal, remove
|
||||
* each node, and run the callback if non-null. This is
|
||||
* used for destroying a tree without paying the cost to
|
||||
* rebalance it. The tree must not be otherwise altered
|
||||
* during traversal.
|
||||
* Args:
|
||||
* tree: Pointer to an initialized red-black tree object.
|
||||
* cb : Callback function, which, if non-null, is called for each node
|
||||
* during iteration. There is no way to stop iteration once it
|
||||
* has begun.
|
||||
* arg : Opaque pointer passed to cb().
|
||||
*/
|
||||
#define rb_gen(a_attr, a_prefix, a_rbt_type, a_type, a_field, a_cmp) \
|
||||
a_attr void \
|
||||
a_prefix##new(a_rbt_type *rbtree) { \
|
||||
rb_new(a_type, a_field, rbtree); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_prefix##empty(a_rbt_type *rbtree) { \
|
||||
return (rbtree->rbt_root == NULL); \
|
||||
} \
|
||||
a_attr a_type * \
|
||||
a_prefix##first(a_rbt_type *rbtree) { \
|
||||
a_type *ret; \
|
||||
rbtn_first(a_type, a_field, rbtree, rbtree->rbt_root, ret); \
|
||||
if (ret == &rbtree->rbt_nil) { \
|
||||
ret = NULL; \
|
||||
} \
|
||||
return (ret); \
|
||||
} \
|
||||
a_attr a_type * \
|
||||
a_prefix##last(a_rbt_type *rbtree) { \
|
||||
a_type *ret; \
|
||||
rbtn_last(a_type, a_field, rbtree, rbtree->rbt_root, ret); \
|
||||
if (ret == &rbtree->rbt_nil) { \
|
||||
ret = NULL; \
|
||||
} \
|
||||
return (ret); \
|
||||
} \
|
||||
a_attr a_type * \
|
||||
a_prefix##next(a_rbt_type *rbtree, a_type *node) { \
|
||||
a_type *ret; \
|
||||
if (rbtn_right_get(a_type, a_field, node) != NULL) { \
|
||||
if (rbtn_right_get(a_type, a_field, node) != &rbtree->rbt_nil) { \
|
||||
rbtn_first(a_type, a_field, rbtree, rbtn_right_get(a_type, \
|
||||
a_field, node), ret); \
|
||||
} else { \
|
||||
a_type *tnode = rbtree->rbt_root; \
|
||||
assert(tnode != NULL); \
|
||||
ret = NULL; \
|
||||
assert(tnode != &rbtree->rbt_nil); \
|
||||
ret = &rbtree->rbt_nil; \
|
||||
while (true) { \
|
||||
int cmp = (a_cmp)(node, tnode); \
|
||||
if (cmp < 0) { \
|
||||
@@ -376,21 +347,24 @@ a_prefix##next(a_rbt_type *rbtree, a_type *node) { \
|
||||
} else { \
|
||||
break; \
|
||||
} \
|
||||
assert(tnode != NULL); \
|
||||
assert(tnode != &rbtree->rbt_nil); \
|
||||
} \
|
||||
} \
|
||||
if (ret == &rbtree->rbt_nil) { \
|
||||
ret = (NULL); \
|
||||
} \
|
||||
return (ret); \
|
||||
} \
|
||||
a_attr a_type * \
|
||||
a_prefix##prev(a_rbt_type *rbtree, a_type *node) { \
|
||||
a_type *ret; \
|
||||
if (rbtn_left_get(a_type, a_field, node) != NULL) { \
|
||||
if (rbtn_left_get(a_type, a_field, node) != &rbtree->rbt_nil) { \
|
||||
rbtn_last(a_type, a_field, rbtree, rbtn_left_get(a_type, \
|
||||
a_field, node), ret); \
|
||||
} else { \
|
||||
a_type *tnode = rbtree->rbt_root; \
|
||||
assert(tnode != NULL); \
|
||||
ret = NULL; \
|
||||
assert(tnode != &rbtree->rbt_nil); \
|
||||
ret = &rbtree->rbt_nil; \
|
||||
while (true) { \
|
||||
int cmp = (a_cmp)(node, tnode); \
|
||||
if (cmp < 0) { \
|
||||
@@ -401,17 +375,20 @@ a_prefix##prev(a_rbt_type *rbtree, a_type *node) { \
|
||||
} else { \
|
||||
break; \
|
||||
} \
|
||||
assert(tnode != NULL); \
|
||||
assert(tnode != &rbtree->rbt_nil); \
|
||||
} \
|
||||
} \
|
||||
if (ret == &rbtree->rbt_nil) { \
|
||||
ret = (NULL); \
|
||||
} \
|
||||
return (ret); \
|
||||
} \
|
||||
a_attr a_type * \
|
||||
a_prefix##search(a_rbt_type *rbtree, const a_type *key) { \
|
||||
a_prefix##search(a_rbt_type *rbtree, a_type *key) { \
|
||||
a_type *ret; \
|
||||
int cmp; \
|
||||
ret = rbtree->rbt_root; \
|
||||
while (ret != NULL \
|
||||
while (ret != &rbtree->rbt_nil \
|
||||
&& (cmp = (a_cmp)(key, ret)) != 0) { \
|
||||
if (cmp < 0) { \
|
||||
ret = rbtn_left_get(a_type, a_field, ret); \
|
||||
@@ -419,14 +396,17 @@ a_prefix##search(a_rbt_type *rbtree, const a_type *key) { \
|
||||
ret = rbtn_right_get(a_type, a_field, ret); \
|
||||
} \
|
||||
} \
|
||||
if (ret == &rbtree->rbt_nil) { \
|
||||
ret = (NULL); \
|
||||
} \
|
||||
return (ret); \
|
||||
} \
|
||||
a_attr a_type * \
|
||||
a_prefix##nsearch(a_rbt_type *rbtree, const a_type *key) { \
|
||||
a_prefix##nsearch(a_rbt_type *rbtree, a_type *key) { \
|
||||
a_type *ret; \
|
||||
a_type *tnode = rbtree->rbt_root; \
|
||||
ret = NULL; \
|
||||
while (tnode != NULL) { \
|
||||
ret = &rbtree->rbt_nil; \
|
||||
while (tnode != &rbtree->rbt_nil) { \
|
||||
int cmp = (a_cmp)(key, tnode); \
|
||||
if (cmp < 0) { \
|
||||
ret = tnode; \
|
||||
@@ -438,14 +418,17 @@ a_prefix##nsearch(a_rbt_type *rbtree, const a_type *key) { \
|
||||
break; \
|
||||
} \
|
||||
} \
|
||||
if (ret == &rbtree->rbt_nil) { \
|
||||
ret = (NULL); \
|
||||
} \
|
||||
return (ret); \
|
||||
} \
|
||||
a_attr a_type * \
|
||||
a_prefix##psearch(a_rbt_type *rbtree, const a_type *key) { \
|
||||
a_prefix##psearch(a_rbt_type *rbtree, a_type *key) { \
|
||||
a_type *ret; \
|
||||
a_type *tnode = rbtree->rbt_root; \
|
||||
ret = NULL; \
|
||||
while (tnode != NULL) { \
|
||||
ret = &rbtree->rbt_nil; \
|
||||
while (tnode != &rbtree->rbt_nil) { \
|
||||
int cmp = (a_cmp)(key, tnode); \
|
||||
if (cmp < 0) { \
|
||||
tnode = rbtn_left_get(a_type, a_field, tnode); \
|
||||
@@ -457,6 +440,9 @@ a_prefix##psearch(a_rbt_type *rbtree, const a_type *key) { \
|
||||
break; \
|
||||
} \
|
||||
} \
|
||||
if (ret == &rbtree->rbt_nil) { \
|
||||
ret = (NULL); \
|
||||
} \
|
||||
return (ret); \
|
||||
} \
|
||||
a_attr void \
|
||||
@@ -468,7 +454,7 @@ a_prefix##insert(a_rbt_type *rbtree, a_type *node) { \
|
||||
rbt_node_new(a_type, a_field, rbtree, node); \
|
||||
/* Wind. */ \
|
||||
path->node = rbtree->rbt_root; \
|
||||
for (pathp = path; pathp->node != NULL; pathp++) { \
|
||||
for (pathp = path; pathp->node != &rbtree->rbt_nil; pathp++) { \
|
||||
int cmp = pathp->cmp = a_cmp(node, pathp->node); \
|
||||
assert(cmp != 0); \
|
||||
if (cmp < 0) { \
|
||||
@@ -488,8 +474,7 @@ a_prefix##insert(a_rbt_type *rbtree, a_type *node) { \
|
||||
rbtn_left_set(a_type, a_field, cnode, left); \
|
||||
if (rbtn_red_get(a_type, a_field, left)) { \
|
||||
a_type *leftleft = rbtn_left_get(a_type, a_field, left);\
|
||||
if (leftleft != NULL && rbtn_red_get(a_type, a_field, \
|
||||
leftleft)) { \
|
||||
if (rbtn_red_get(a_type, a_field, leftleft)) { \
|
||||
/* Fix up 4-node. */ \
|
||||
a_type *tnode; \
|
||||
rbtn_black_set(a_type, a_field, leftleft); \
|
||||
@@ -504,8 +489,7 @@ a_prefix##insert(a_rbt_type *rbtree, a_type *node) { \
|
||||
rbtn_right_set(a_type, a_field, cnode, right); \
|
||||
if (rbtn_red_get(a_type, a_field, right)) { \
|
||||
a_type *left = rbtn_left_get(a_type, a_field, cnode); \
|
||||
if (left != NULL && rbtn_red_get(a_type, a_field, \
|
||||
left)) { \
|
||||
if (rbtn_red_get(a_type, a_field, left)) { \
|
||||
/* Split 4-node. */ \
|
||||
rbtn_black_set(a_type, a_field, left); \
|
||||
rbtn_black_set(a_type, a_field, right); \
|
||||
@@ -538,7 +522,7 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \
|
||||
/* Wind. */ \
|
||||
nodep = NULL; /* Silence compiler warning. */ \
|
||||
path->node = rbtree->rbt_root; \
|
||||
for (pathp = path; pathp->node != NULL; pathp++) { \
|
||||
for (pathp = path; pathp->node != &rbtree->rbt_nil; pathp++) { \
|
||||
int cmp = pathp->cmp = a_cmp(node, pathp->node); \
|
||||
if (cmp < 0) { \
|
||||
pathp[1].node = rbtn_left_get(a_type, a_field, \
|
||||
@@ -550,7 +534,7 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \
|
||||
/* Find node's successor, in preparation for swap. */ \
|
||||
pathp->cmp = 1; \
|
||||
nodep = pathp; \
|
||||
for (pathp++; pathp->node != NULL; \
|
||||
for (pathp++; pathp->node != &rbtree->rbt_nil; \
|
||||
pathp++) { \
|
||||
pathp->cmp = -1; \
|
||||
pathp[1].node = rbtn_left_get(a_type, a_field, \
|
||||
@@ -593,10 +577,10 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \
|
||||
} \
|
||||
} else { \
|
||||
a_type *left = rbtn_left_get(a_type, a_field, node); \
|
||||
if (left != NULL) { \
|
||||
if (left != &rbtree->rbt_nil) { \
|
||||
/* node has no successor, but it has a left child. */\
|
||||
/* Splice node out, without losing the left child. */\
|
||||
assert(!rbtn_red_get(a_type, a_field, node)); \
|
||||
assert(rbtn_red_get(a_type, a_field, node) == false); \
|
||||
assert(rbtn_red_get(a_type, a_field, left)); \
|
||||
rbtn_black_set(a_type, a_field, left); \
|
||||
if (pathp == path) { \
|
||||
@@ -613,32 +597,34 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \
|
||||
return; \
|
||||
} else if (pathp == path) { \
|
||||
/* The tree only contained one node. */ \
|
||||
rbtree->rbt_root = NULL; \
|
||||
rbtree->rbt_root = &rbtree->rbt_nil; \
|
||||
return; \
|
||||
} \
|
||||
} \
|
||||
if (rbtn_red_get(a_type, a_field, pathp->node)) { \
|
||||
/* Prune red node, which requires no fixup. */ \
|
||||
assert(pathp[-1].cmp < 0); \
|
||||
rbtn_left_set(a_type, a_field, pathp[-1].node, NULL); \
|
||||
rbtn_left_set(a_type, a_field, pathp[-1].node, \
|
||||
&rbtree->rbt_nil); \
|
||||
return; \
|
||||
} \
|
||||
/* The node to be pruned is black, so unwind until balance is */\
|
||||
/* restored. */\
|
||||
pathp->node = NULL; \
|
||||
pathp->node = &rbtree->rbt_nil; \
|
||||
for (pathp--; (uintptr_t)pathp >= (uintptr_t)path; pathp--) { \
|
||||
assert(pathp->cmp != 0); \
|
||||
if (pathp->cmp < 0) { \
|
||||
rbtn_left_set(a_type, a_field, pathp->node, \
|
||||
pathp[1].node); \
|
||||
assert(rbtn_red_get(a_type, a_field, pathp[1].node) \
|
||||
== false); \
|
||||
if (rbtn_red_get(a_type, a_field, pathp->node)) { \
|
||||
a_type *right = rbtn_right_get(a_type, a_field, \
|
||||
pathp->node); \
|
||||
a_type *rightleft = rbtn_left_get(a_type, a_field, \
|
||||
right); \
|
||||
a_type *tnode; \
|
||||
if (rightleft != NULL && rbtn_red_get(a_type, a_field, \
|
||||
rightleft)) { \
|
||||
if (rbtn_red_get(a_type, a_field, rightleft)) { \
|
||||
/* In the following diagrams, ||, //, and \\ */\
|
||||
/* indicate the path to the removed node. */\
|
||||
/* */\
|
||||
@@ -681,8 +667,7 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \
|
||||
pathp->node); \
|
||||
a_type *rightleft = rbtn_left_get(a_type, a_field, \
|
||||
right); \
|
||||
if (rightleft != NULL && rbtn_red_get(a_type, a_field, \
|
||||
rightleft)) { \
|
||||
if (rbtn_red_get(a_type, a_field, rightleft)) { \
|
||||
/* || */\
|
||||
/* pathp(b) */\
|
||||
/* // \ */\
|
||||
@@ -696,7 +681,7 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \
|
||||
rbtn_rotate_left(a_type, a_field, pathp->node, \
|
||||
tnode); \
|
||||
/* Balance restored, but rotation modified */\
|
||||
/* subtree root, which may actually be the tree */\
|
||||
/* subree root, which may actually be the tree */\
|
||||
/* root. */\
|
||||
if (pathp == path) { \
|
||||
/* Set root. */ \
|
||||
@@ -736,8 +721,7 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \
|
||||
left); \
|
||||
a_type *leftrightleft = rbtn_left_get(a_type, a_field, \
|
||||
leftright); \
|
||||
if (leftrightleft != NULL && rbtn_red_get(a_type, \
|
||||
a_field, leftrightleft)) { \
|
||||
if (rbtn_red_get(a_type, a_field, leftrightleft)) { \
|
||||
/* || */\
|
||||
/* pathp(b) */\
|
||||
/* / \\ */\
|
||||
@@ -763,7 +747,7 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \
|
||||
/* (b) */\
|
||||
/* / */\
|
||||
/* (b) */\
|
||||
assert(leftright != NULL); \
|
||||
assert(leftright != &rbtree->rbt_nil); \
|
||||
rbtn_red_set(a_type, a_field, leftright); \
|
||||
rbtn_rotate_right(a_type, a_field, pathp->node, \
|
||||
tnode); \
|
||||
@@ -786,8 +770,7 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \
|
||||
return; \
|
||||
} else if (rbtn_red_get(a_type, a_field, pathp->node)) { \
|
||||
a_type *leftleft = rbtn_left_get(a_type, a_field, left);\
|
||||
if (leftleft != NULL && rbtn_red_get(a_type, a_field, \
|
||||
leftleft)) { \
|
||||
if (rbtn_red_get(a_type, a_field, leftleft)) { \
|
||||
/* || */\
|
||||
/* pathp(r) */\
|
||||
/* / \\ */\
|
||||
@@ -825,8 +808,7 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \
|
||||
} \
|
||||
} else { \
|
||||
a_type *leftleft = rbtn_left_get(a_type, a_field, left);\
|
||||
if (leftleft != NULL && rbtn_red_get(a_type, a_field, \
|
||||
leftleft)) { \
|
||||
if (rbtn_red_get(a_type, a_field, leftleft)) { \
|
||||
/* || */\
|
||||
/* pathp(b) */\
|
||||
/* / \\ */\
|
||||
@@ -867,18 +849,18 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \
|
||||
} \
|
||||
/* Set root. */ \
|
||||
rbtree->rbt_root = path->node; \
|
||||
assert(!rbtn_red_get(a_type, a_field, rbtree->rbt_root)); \
|
||||
assert(rbtn_red_get(a_type, a_field, rbtree->rbt_root) == false); \
|
||||
} \
|
||||
a_attr a_type * \
|
||||
a_prefix##iter_recurse(a_rbt_type *rbtree, a_type *node, \
|
||||
a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg) { \
|
||||
if (node == NULL) { \
|
||||
return (NULL); \
|
||||
if (node == &rbtree->rbt_nil) { \
|
||||
return (&rbtree->rbt_nil); \
|
||||
} else { \
|
||||
a_type *ret; \
|
||||
if ((ret = a_prefix##iter_recurse(rbtree, rbtn_left_get(a_type, \
|
||||
a_field, node), cb, arg)) != NULL || (ret = cb(rbtree, node, \
|
||||
arg)) != NULL) { \
|
||||
a_field, node), cb, arg)) != &rbtree->rbt_nil \
|
||||
|| (ret = cb(rbtree, node, arg)) != NULL) { \
|
||||
return (ret); \
|
||||
} \
|
||||
return (a_prefix##iter_recurse(rbtree, rbtn_right_get(a_type, \
|
||||
@@ -892,8 +874,8 @@ a_prefix##iter_start(a_rbt_type *rbtree, a_type *start, a_type *node, \
|
||||
if (cmp < 0) { \
|
||||
a_type *ret; \
|
||||
if ((ret = a_prefix##iter_start(rbtree, start, \
|
||||
rbtn_left_get(a_type, a_field, node), cb, arg)) != NULL || \
|
||||
(ret = cb(rbtree, node, arg)) != NULL) { \
|
||||
rbtn_left_get(a_type, a_field, node), cb, arg)) != \
|
||||
&rbtree->rbt_nil || (ret = cb(rbtree, node, arg)) != NULL) { \
|
||||
return (ret); \
|
||||
} \
|
||||
return (a_prefix##iter_recurse(rbtree, rbtn_right_get(a_type, \
|
||||
@@ -920,18 +902,21 @@ a_prefix##iter(a_rbt_type *rbtree, a_type *start, a_type *(*cb)( \
|
||||
} else { \
|
||||
ret = a_prefix##iter_recurse(rbtree, rbtree->rbt_root, cb, arg);\
|
||||
} \
|
||||
if (ret == &rbtree->rbt_nil) { \
|
||||
ret = NULL; \
|
||||
} \
|
||||
return (ret); \
|
||||
} \
|
||||
a_attr a_type * \
|
||||
a_prefix##reverse_iter_recurse(a_rbt_type *rbtree, a_type *node, \
|
||||
a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg) { \
|
||||
if (node == NULL) { \
|
||||
return (NULL); \
|
||||
if (node == &rbtree->rbt_nil) { \
|
||||
return (&rbtree->rbt_nil); \
|
||||
} else { \
|
||||
a_type *ret; \
|
||||
if ((ret = a_prefix##reverse_iter_recurse(rbtree, \
|
||||
rbtn_right_get(a_type, a_field, node), cb, arg)) != NULL || \
|
||||
(ret = cb(rbtree, node, arg)) != NULL) { \
|
||||
rbtn_right_get(a_type, a_field, node), cb, arg)) != \
|
||||
&rbtree->rbt_nil || (ret = cb(rbtree, node, arg)) != NULL) { \
|
||||
return (ret); \
|
||||
} \
|
||||
return (a_prefix##reverse_iter_recurse(rbtree, \
|
||||
@@ -946,8 +931,8 @@ a_prefix##reverse_iter_start(a_rbt_type *rbtree, a_type *start, \
|
||||
if (cmp > 0) { \
|
||||
a_type *ret; \
|
||||
if ((ret = a_prefix##reverse_iter_start(rbtree, start, \
|
||||
rbtn_right_get(a_type, a_field, node), cb, arg)) != NULL || \
|
||||
(ret = cb(rbtree, node, arg)) != NULL) { \
|
||||
rbtn_right_get(a_type, a_field, node), cb, arg)) != \
|
||||
&rbtree->rbt_nil || (ret = cb(rbtree, node, arg)) != NULL) { \
|
||||
return (ret); \
|
||||
} \
|
||||
return (a_prefix##reverse_iter_recurse(rbtree, \
|
||||
@@ -975,29 +960,10 @@ a_prefix##reverse_iter(a_rbt_type *rbtree, a_type *start, \
|
||||
ret = a_prefix##reverse_iter_recurse(rbtree, rbtree->rbt_root, \
|
||||
cb, arg); \
|
||||
} \
|
||||
if (ret == &rbtree->rbt_nil) { \
|
||||
ret = NULL; \
|
||||
} \
|
||||
return (ret); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_prefix##destroy_recurse(a_rbt_type *rbtree, a_type *node, void (*cb)( \
|
||||
a_type *, void *), void *arg) { \
|
||||
if (node == NULL) { \
|
||||
return; \
|
||||
} \
|
||||
a_prefix##destroy_recurse(rbtree, rbtn_left_get(a_type, a_field, \
|
||||
node), cb, arg); \
|
||||
rbtn_left_set(a_type, a_field, (node), NULL); \
|
||||
a_prefix##destroy_recurse(rbtree, rbtn_right_get(a_type, a_field, \
|
||||
node), cb, arg); \
|
||||
rbtn_right_set(a_type, a_field, (node), NULL); \
|
||||
if (cb) { \
|
||||
cb(node, arg); \
|
||||
} \
|
||||
} \
|
||||
a_attr void \
|
||||
a_prefix##destroy(a_rbt_type *rbtree, void (*cb)(a_type *, void *), \
|
||||
void *arg) { \
|
||||
a_prefix##destroy_recurse(rbtree, rbtree->rbt_root, cb, arg); \
|
||||
rbtree->rbt_root = NULL; \
|
||||
}
|
||||
|
||||
#endif /* RB_H_ */
|
||||
|
||||
@@ -1,364 +1,170 @@
|
||||
/*
|
||||
* This radix tree implementation is tailored to the singular purpose of
|
||||
* associating metadata with chunks that are currently owned by jemalloc.
|
||||
* tracking which chunks are currently owned by jemalloc. This functionality
|
||||
* is mandatory for OS X, where jemalloc must be able to respond to object
|
||||
* ownership queries.
|
||||
*
|
||||
*******************************************************************************
|
||||
*/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
typedef struct rtree_node_elm_s rtree_node_elm_t;
|
||||
typedef struct rtree_level_s rtree_level_t;
|
||||
typedef struct rtree_s rtree_t;
|
||||
|
||||
/*
|
||||
* RTREE_BITS_PER_LEVEL must be a power of two that is no larger than the
|
||||
* machine address width.
|
||||
* Size of each radix tree node (must be a power of 2). This impacts tree
|
||||
* depth.
|
||||
*/
|
||||
#define LG_RTREE_BITS_PER_LEVEL 4
|
||||
#define RTREE_BITS_PER_LEVEL (1U << LG_RTREE_BITS_PER_LEVEL)
|
||||
/* Maximum rtree height. */
|
||||
#define RTREE_HEIGHT_MAX \
|
||||
((1U << (LG_SIZEOF_PTR+3)) / RTREE_BITS_PER_LEVEL)
|
||||
#define RTREE_NODESIZE (1U << 16)
|
||||
|
||||
/* Used for two-stage lock-free node initialization. */
|
||||
#define RTREE_NODE_INITIALIZING ((rtree_node_elm_t *)0x1)
|
||||
|
||||
/*
|
||||
* The node allocation callback function's argument is the number of contiguous
|
||||
* rtree_node_elm_t structures to allocate, and the resulting memory must be
|
||||
* zeroed.
|
||||
*/
|
||||
typedef rtree_node_elm_t *(rtree_node_alloc_t)(size_t);
|
||||
typedef void (rtree_node_dalloc_t)(rtree_node_elm_t *);
|
||||
typedef void *(rtree_alloc_t)(size_t);
|
||||
typedef void (rtree_dalloc_t)(void *);
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
|
||||
struct rtree_node_elm_s {
|
||||
union {
|
||||
void *pun;
|
||||
rtree_node_elm_t *child;
|
||||
extent_node_t *val;
|
||||
};
|
||||
};
|
||||
|
||||
struct rtree_level_s {
|
||||
/*
|
||||
* A non-NULL subtree points to a subtree rooted along the hypothetical
|
||||
* path to the leaf node corresponding to key 0. Depending on what keys
|
||||
* have been used to store to the tree, an arbitrary combination of
|
||||
* subtree pointers may remain NULL.
|
||||
*
|
||||
* Suppose keys comprise 48 bits, and LG_RTREE_BITS_PER_LEVEL is 4.
|
||||
* This results in a 3-level tree, and the leftmost leaf can be directly
|
||||
* accessed via subtrees[2], the subtree prefixed by 0x0000 (excluding
|
||||
* 0x00000000) can be accessed via subtrees[1], and the remainder of the
|
||||
* tree can be accessed via subtrees[0].
|
||||
*
|
||||
* levels[0] : [<unused> | 0x0001******** | 0x0002******** | ...]
|
||||
*
|
||||
* levels[1] : [<unused> | 0x00000001**** | 0x00000002**** | ... ]
|
||||
*
|
||||
* levels[2] : [val(0x000000000000) | val(0x000000000001) | ...]
|
||||
*
|
||||
* This has practical implications on x64, which currently uses only the
|
||||
* lower 47 bits of virtual address space in userland, thus leaving
|
||||
* subtrees[0] unused and avoiding a level of tree traversal.
|
||||
*/
|
||||
union {
|
||||
void *subtree_pun;
|
||||
rtree_node_elm_t *subtree;
|
||||
};
|
||||
/* Number of key bits distinguished by this level. */
|
||||
unsigned bits;
|
||||
/*
|
||||
* Cumulative number of key bits distinguished by traversing to
|
||||
* corresponding tree level.
|
||||
*/
|
||||
unsigned cumbits;
|
||||
};
|
||||
|
||||
struct rtree_s {
|
||||
rtree_node_alloc_t *alloc;
|
||||
rtree_node_dalloc_t *dalloc;
|
||||
unsigned height;
|
||||
/*
|
||||
* Precomputed table used to convert from the number of leading 0 key
|
||||
* bits to which subtree level to start at.
|
||||
*/
|
||||
unsigned start_level[RTREE_HEIGHT_MAX];
|
||||
rtree_level_t levels[RTREE_HEIGHT_MAX];
|
||||
rtree_alloc_t *alloc;
|
||||
rtree_dalloc_t *dalloc;
|
||||
malloc_mutex_t mutex;
|
||||
void **root;
|
||||
unsigned height;
|
||||
unsigned level2bits[1]; /* Dynamically sized. */
|
||||
};
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
bool rtree_new(rtree_t *rtree, unsigned bits, rtree_node_alloc_t *alloc,
|
||||
rtree_node_dalloc_t *dalloc);
|
||||
rtree_t *rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc);
|
||||
void rtree_delete(rtree_t *rtree);
|
||||
rtree_node_elm_t *rtree_subtree_read_hard(rtree_t *rtree,
|
||||
unsigned level);
|
||||
rtree_node_elm_t *rtree_child_read_hard(rtree_t *rtree,
|
||||
rtree_node_elm_t *elm, unsigned level);
|
||||
void rtree_prefork(rtree_t *rtree);
|
||||
void rtree_postfork_parent(rtree_t *rtree);
|
||||
void rtree_postfork_child(rtree_t *rtree);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
unsigned rtree_start_level(rtree_t *rtree, uintptr_t key);
|
||||
uintptr_t rtree_subkey(rtree_t *rtree, uintptr_t key, unsigned level);
|
||||
|
||||
bool rtree_node_valid(rtree_node_elm_t *node);
|
||||
rtree_node_elm_t *rtree_child_tryread(rtree_node_elm_t *elm,
|
||||
bool dependent);
|
||||
rtree_node_elm_t *rtree_child_read(rtree_t *rtree, rtree_node_elm_t *elm,
|
||||
unsigned level, bool dependent);
|
||||
extent_node_t *rtree_val_read(rtree_t *rtree, rtree_node_elm_t *elm,
|
||||
bool dependent);
|
||||
void rtree_val_write(rtree_t *rtree, rtree_node_elm_t *elm,
|
||||
const extent_node_t *val);
|
||||
rtree_node_elm_t *rtree_subtree_tryread(rtree_t *rtree, unsigned level,
|
||||
bool dependent);
|
||||
rtree_node_elm_t *rtree_subtree_read(rtree_t *rtree, unsigned level,
|
||||
bool dependent);
|
||||
|
||||
extent_node_t *rtree_get(rtree_t *rtree, uintptr_t key, bool dependent);
|
||||
bool rtree_set(rtree_t *rtree, uintptr_t key, const extent_node_t *val);
|
||||
#ifdef JEMALLOC_DEBUG
|
||||
uint8_t rtree_get_locked(rtree_t *rtree, uintptr_t key);
|
||||
#endif
|
||||
uint8_t rtree_get(rtree_t *rtree, uintptr_t key);
|
||||
bool rtree_set(rtree_t *rtree, uintptr_t key, uint8_t val);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_))
|
||||
JEMALLOC_ALWAYS_INLINE unsigned
|
||||
rtree_start_level(rtree_t *rtree, uintptr_t key)
|
||||
{
|
||||
unsigned start_level;
|
||||
|
||||
if (unlikely(key == 0))
|
||||
return (rtree->height - 1);
|
||||
|
||||
start_level = rtree->start_level[lg_floor(key) >>
|
||||
LG_RTREE_BITS_PER_LEVEL];
|
||||
assert(start_level < rtree->height);
|
||||
return (start_level);
|
||||
#define RTREE_GET_GENERATE(f) \
|
||||
/* The least significant bits of the key are ignored. */ \
|
||||
JEMALLOC_INLINE uint8_t \
|
||||
f(rtree_t *rtree, uintptr_t key) \
|
||||
{ \
|
||||
uint8_t ret; \
|
||||
uintptr_t subkey; \
|
||||
unsigned i, lshift, height, bits; \
|
||||
void **node, **child; \
|
||||
\
|
||||
RTREE_LOCK(&rtree->mutex); \
|
||||
for (i = lshift = 0, height = rtree->height, node = rtree->root;\
|
||||
i < height - 1; \
|
||||
i++, lshift += bits, node = child) { \
|
||||
bits = rtree->level2bits[i]; \
|
||||
subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR + \
|
||||
3)) - bits); \
|
||||
child = (void**)node[subkey]; \
|
||||
if (child == NULL) { \
|
||||
RTREE_UNLOCK(&rtree->mutex); \
|
||||
return (0); \
|
||||
} \
|
||||
} \
|
||||
\
|
||||
/* \
|
||||
* node is a leaf, so it contains values rather than node \
|
||||
* pointers. \
|
||||
*/ \
|
||||
bits = rtree->level2bits[i]; \
|
||||
subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - \
|
||||
bits); \
|
||||
{ \
|
||||
uint8_t *leaf = (uint8_t *)node; \
|
||||
ret = leaf[subkey]; \
|
||||
} \
|
||||
RTREE_UNLOCK(&rtree->mutex); \
|
||||
\
|
||||
RTREE_GET_VALIDATE \
|
||||
return (ret); \
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE uintptr_t
|
||||
rtree_subkey(rtree_t *rtree, uintptr_t key, unsigned level)
|
||||
{
|
||||
|
||||
return ((key >> ((ZU(1) << (LG_SIZEOF_PTR+3)) -
|
||||
rtree->levels[level].cumbits)) & ((ZU(1) <<
|
||||
rtree->levels[level].bits) - 1));
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
rtree_node_valid(rtree_node_elm_t *node)
|
||||
{
|
||||
|
||||
return ((uintptr_t)node > (uintptr_t)RTREE_NODE_INITIALIZING);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE rtree_node_elm_t *
|
||||
rtree_child_tryread(rtree_node_elm_t *elm, bool dependent)
|
||||
{
|
||||
rtree_node_elm_t *child;
|
||||
|
||||
/* Double-checked read (first read may be stale. */
|
||||
child = elm->child;
|
||||
if (!dependent && !rtree_node_valid(child))
|
||||
child = atomic_read_p(&elm->pun);
|
||||
assert(!dependent || child != NULL);
|
||||
return (child);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE rtree_node_elm_t *
|
||||
rtree_child_read(rtree_t *rtree, rtree_node_elm_t *elm, unsigned level,
|
||||
bool dependent)
|
||||
{
|
||||
rtree_node_elm_t *child;
|
||||
|
||||
child = rtree_child_tryread(elm, dependent);
|
||||
if (!dependent && unlikely(!rtree_node_valid(child)))
|
||||
child = rtree_child_read_hard(rtree, elm, level);
|
||||
assert(!dependent || child != NULL);
|
||||
return (child);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE extent_node_t *
|
||||
rtree_val_read(rtree_t *rtree, rtree_node_elm_t *elm, bool dependent)
|
||||
{
|
||||
|
||||
if (dependent) {
|
||||
/*
|
||||
* Reading a val on behalf of a pointer to a valid allocation is
|
||||
* guaranteed to be a clean read even without synchronization,
|
||||
* because the rtree update became visible in memory before the
|
||||
* pointer came into existence.
|
||||
*/
|
||||
return (elm->val);
|
||||
} else {
|
||||
/*
|
||||
* An arbitrary read, e.g. on behalf of ivsalloc(), may not be
|
||||
* dependent on a previous rtree write, which means a stale read
|
||||
* could result if synchronization were omitted here.
|
||||
*/
|
||||
return (atomic_read_p(&elm->pun));
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
rtree_val_write(rtree_t *rtree, rtree_node_elm_t *elm, const extent_node_t *val)
|
||||
{
|
||||
|
||||
atomic_write_p(&elm->pun, val);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE rtree_node_elm_t *
|
||||
rtree_subtree_tryread(rtree_t *rtree, unsigned level, bool dependent)
|
||||
{
|
||||
rtree_node_elm_t *subtree;
|
||||
|
||||
/* Double-checked read (first read may be stale. */
|
||||
subtree = rtree->levels[level].subtree;
|
||||
if (!dependent && unlikely(!rtree_node_valid(subtree)))
|
||||
subtree = atomic_read_p(&rtree->levels[level].subtree_pun);
|
||||
assert(!dependent || subtree != NULL);
|
||||
return (subtree);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE rtree_node_elm_t *
|
||||
rtree_subtree_read(rtree_t *rtree, unsigned level, bool dependent)
|
||||
{
|
||||
rtree_node_elm_t *subtree;
|
||||
|
||||
subtree = rtree_subtree_tryread(rtree, level, dependent);
|
||||
if (!dependent && unlikely(!rtree_node_valid(subtree)))
|
||||
subtree = rtree_subtree_read_hard(rtree, level);
|
||||
assert(!dependent || subtree != NULL);
|
||||
return (subtree);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE extent_node_t *
|
||||
rtree_get(rtree_t *rtree, uintptr_t key, bool dependent)
|
||||
{
|
||||
uintptr_t subkey;
|
||||
unsigned start_level;
|
||||
rtree_node_elm_t *node;
|
||||
|
||||
start_level = rtree_start_level(rtree, key);
|
||||
|
||||
node = rtree_subtree_tryread(rtree, start_level, dependent);
|
||||
#define RTREE_GET_BIAS (RTREE_HEIGHT_MAX - rtree->height)
|
||||
switch (start_level + RTREE_GET_BIAS) {
|
||||
#define RTREE_GET_SUBTREE(level) \
|
||||
case level: \
|
||||
assert(level < (RTREE_HEIGHT_MAX-1)); \
|
||||
if (!dependent && unlikely(!rtree_node_valid(node))) \
|
||||
return (NULL); \
|
||||
subkey = rtree_subkey(rtree, key, level - \
|
||||
RTREE_GET_BIAS); \
|
||||
node = rtree_child_tryread(&node[subkey], dependent); \
|
||||
/* Fall through. */
|
||||
#define RTREE_GET_LEAF(level) \
|
||||
case level: \
|
||||
assert(level == (RTREE_HEIGHT_MAX-1)); \
|
||||
if (!dependent && unlikely(!rtree_node_valid(node))) \
|
||||
return (NULL); \
|
||||
subkey = rtree_subkey(rtree, key, level - \
|
||||
RTREE_GET_BIAS); \
|
||||
/* \
|
||||
* node is a leaf, so it contains values rather than \
|
||||
* child pointers. \
|
||||
*/ \
|
||||
return (rtree_val_read(rtree, &node[subkey], \
|
||||
dependent));
|
||||
#if RTREE_HEIGHT_MAX > 1
|
||||
RTREE_GET_SUBTREE(0)
|
||||
#ifdef JEMALLOC_DEBUG
|
||||
# define RTREE_LOCK(l) malloc_mutex_lock(l)
|
||||
# define RTREE_UNLOCK(l) malloc_mutex_unlock(l)
|
||||
# define RTREE_GET_VALIDATE
|
||||
RTREE_GET_GENERATE(rtree_get_locked)
|
||||
# undef RTREE_LOCK
|
||||
# undef RTREE_UNLOCK
|
||||
# undef RTREE_GET_VALIDATE
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 2
|
||||
RTREE_GET_SUBTREE(1)
|
||||
|
||||
#define RTREE_LOCK(l)
|
||||
#define RTREE_UNLOCK(l)
|
||||
#ifdef JEMALLOC_DEBUG
|
||||
/*
|
||||
* Suppose that it were possible for a jemalloc-allocated chunk to be
|
||||
* munmap()ped, followed by a different allocator in another thread re-using
|
||||
* overlapping virtual memory, all without invalidating the cached rtree
|
||||
* value. The result would be a false positive (the rtree would claim that
|
||||
* jemalloc owns memory that it had actually discarded). This scenario
|
||||
* seems impossible, but the following assertion is a prudent sanity check.
|
||||
*/
|
||||
# define RTREE_GET_VALIDATE \
|
||||
assert(rtree_get_locked(rtree, key) == ret);
|
||||
#else
|
||||
# define RTREE_GET_VALIDATE
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 3
|
||||
RTREE_GET_SUBTREE(2)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 4
|
||||
RTREE_GET_SUBTREE(3)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 5
|
||||
RTREE_GET_SUBTREE(4)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 6
|
||||
RTREE_GET_SUBTREE(5)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 7
|
||||
RTREE_GET_SUBTREE(6)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 8
|
||||
RTREE_GET_SUBTREE(7)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 9
|
||||
RTREE_GET_SUBTREE(8)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 10
|
||||
RTREE_GET_SUBTREE(9)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 11
|
||||
RTREE_GET_SUBTREE(10)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 12
|
||||
RTREE_GET_SUBTREE(11)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 13
|
||||
RTREE_GET_SUBTREE(12)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 14
|
||||
RTREE_GET_SUBTREE(13)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 15
|
||||
RTREE_GET_SUBTREE(14)
|
||||
#endif
|
||||
#if RTREE_HEIGHT_MAX > 16
|
||||
# error Unsupported RTREE_HEIGHT_MAX
|
||||
#endif
|
||||
RTREE_GET_LEAF(RTREE_HEIGHT_MAX-1)
|
||||
#undef RTREE_GET_SUBTREE
|
||||
#undef RTREE_GET_LEAF
|
||||
default: not_reached();
|
||||
}
|
||||
#undef RTREE_GET_BIAS
|
||||
not_reached();
|
||||
}
|
||||
RTREE_GET_GENERATE(rtree_get)
|
||||
#undef RTREE_LOCK
|
||||
#undef RTREE_UNLOCK
|
||||
#undef RTREE_GET_VALIDATE
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
rtree_set(rtree_t *rtree, uintptr_t key, const extent_node_t *val)
|
||||
rtree_set(rtree_t *rtree, uintptr_t key, uint8_t val)
|
||||
{
|
||||
uintptr_t subkey;
|
||||
unsigned i, start_level;
|
||||
rtree_node_elm_t *node, *child;
|
||||
unsigned i, lshift, height, bits;
|
||||
void **node, **child;
|
||||
|
||||
start_level = rtree_start_level(rtree, key);
|
||||
|
||||
node = rtree_subtree_read(rtree, start_level, false);
|
||||
if (node == NULL)
|
||||
return (true);
|
||||
for (i = start_level; /**/; i++, node = child) {
|
||||
subkey = rtree_subkey(rtree, key, i);
|
||||
if (i == rtree->height - 1) {
|
||||
/*
|
||||
* node is a leaf, so it contains values rather than
|
||||
* child pointers.
|
||||
*/
|
||||
rtree_val_write(rtree, &node[subkey], val);
|
||||
return (false);
|
||||
malloc_mutex_lock(&rtree->mutex);
|
||||
for (i = lshift = 0, height = rtree->height, node = rtree->root;
|
||||
i < height - 1;
|
||||
i++, lshift += bits, node = child) {
|
||||
bits = rtree->level2bits[i];
|
||||
subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) -
|
||||
bits);
|
||||
child = (void**)node[subkey];
|
||||
if (child == NULL) {
|
||||
size_t size = ((i + 1 < height - 1) ? sizeof(void *)
|
||||
: (sizeof(uint8_t))) << rtree->level2bits[i+1];
|
||||
child = (void**)rtree->alloc(size);
|
||||
if (child == NULL) {
|
||||
malloc_mutex_unlock(&rtree->mutex);
|
||||
return (true);
|
||||
}
|
||||
memset(child, 0, size);
|
||||
node[subkey] = child;
|
||||
}
|
||||
assert(i + 1 < rtree->height);
|
||||
child = rtree_child_read(rtree, &node[subkey], i, false);
|
||||
if (child == NULL)
|
||||
return (true);
|
||||
}
|
||||
not_reached();
|
||||
|
||||
/* node is a leaf, so it contains values rather than node pointers. */
|
||||
bits = rtree->level2bits[i];
|
||||
subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - bits);
|
||||
{
|
||||
uint8_t *leaf = (uint8_t *)node;
|
||||
leaf[subkey] = val;
|
||||
}
|
||||
malloc_mutex_unlock(&rtree->mutex);
|
||||
|
||||
return (false);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@@ -1,26 +1,17 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Usage: size_classes.sh <lg_qarr> <lg_tmin> <lg_parr> <lg_g>
|
||||
|
||||
# The following limits are chosen such that they cover all supported platforms.
|
||||
|
||||
# Pointer sizes.
|
||||
lg_zarr="2 3"
|
||||
|
||||
# Quanta.
|
||||
lg_qarr=$1
|
||||
# Range of quanta.
|
||||
lg_qmin=3
|
||||
lg_qmax=4
|
||||
|
||||
# The range of tiny size classes is [2^lg_tmin..2^(lg_q-1)].
|
||||
lg_tmin=$2
|
||||
lg_tmin=3
|
||||
|
||||
# Maximum lookup size.
|
||||
lg_kmax=12
|
||||
|
||||
# Page sizes.
|
||||
lg_parr=`echo $3 | tr ',' ' '`
|
||||
|
||||
# Size class group size (number of size classes for each size doubling).
|
||||
lg_g=$4
|
||||
# Range of page sizes.
|
||||
lg_pmin=12
|
||||
lg_pmax=16
|
||||
|
||||
pow2() {
|
||||
e=$1
|
||||
@@ -31,256 +22,68 @@ pow2() {
|
||||
done
|
||||
}
|
||||
|
||||
lg() {
|
||||
x=$1
|
||||
lg_result=0
|
||||
while [ ${x} -gt 1 ] ; do
|
||||
lg_result=$((${lg_result} + 1))
|
||||
x=$((${x} / 2))
|
||||
done
|
||||
}
|
||||
|
||||
size_class() {
|
||||
index=$1
|
||||
lg_grp=$2
|
||||
lg_delta=$3
|
||||
ndelta=$4
|
||||
lg_p=$5
|
||||
lg_kmax=$6
|
||||
|
||||
if [ ${lg_delta} -ge ${lg_p} ] ; then
|
||||
psz="yes"
|
||||
else
|
||||
pow2 ${lg_p}; p=${pow2_result}
|
||||
pow2 ${lg_grp}; grp=${pow2_result}
|
||||
pow2 ${lg_delta}; delta=${pow2_result}
|
||||
sz=$((${grp} + ${delta} * ${ndelta}))
|
||||
npgs=$((${sz} / ${p}))
|
||||
if [ ${sz} -eq $((${npgs} * ${p})) ] ; then
|
||||
psz="yes"
|
||||
else
|
||||
psz="no"
|
||||
fi
|
||||
fi
|
||||
|
||||
lg ${ndelta}; lg_ndelta=${lg_result}; pow2 ${lg_ndelta}
|
||||
if [ ${pow2_result} -lt ${ndelta} ] ; then
|
||||
rem="yes"
|
||||
else
|
||||
rem="no"
|
||||
fi
|
||||
|
||||
lg_size=${lg_grp}
|
||||
if [ $((${lg_delta} + ${lg_ndelta})) -eq ${lg_grp} ] ; then
|
||||
lg_size=$((${lg_grp} + 1))
|
||||
else
|
||||
lg_size=${lg_grp}
|
||||
rem="yes"
|
||||
fi
|
||||
|
||||
if [ ${lg_size} -lt $((${lg_p} + ${lg_g})) ] ; then
|
||||
bin="yes"
|
||||
else
|
||||
bin="no"
|
||||
fi
|
||||
if [ ${lg_size} -lt ${lg_kmax} \
|
||||
-o ${lg_size} -eq ${lg_kmax} -a ${rem} = "no" ] ; then
|
||||
lg_delta_lookup=${lg_delta}
|
||||
else
|
||||
lg_delta_lookup="no"
|
||||
fi
|
||||
printf ' SC(%3d, %6d, %8d, %6d, %3s, %3s, %2s) \\\n' ${index} ${lg_grp} ${lg_delta} ${ndelta} ${psz} ${bin} ${lg_delta_lookup}
|
||||
# Defined upon return:
|
||||
# - psz ("yes" or "no")
|
||||
# - bin ("yes" or "no")
|
||||
# - lg_delta_lookup (${lg_delta} or "no")
|
||||
}
|
||||
|
||||
sep_line() {
|
||||
echo " \\"
|
||||
}
|
||||
|
||||
size_classes() {
|
||||
lg_z=$1
|
||||
lg_q=$2
|
||||
lg_t=$3
|
||||
lg_p=$4
|
||||
lg_g=$5
|
||||
|
||||
pow2 $((${lg_z} + 3)); ptr_bits=${pow2_result}
|
||||
pow2 ${lg_g}; g=${pow2_result}
|
||||
|
||||
echo "#define SIZE_CLASSES \\"
|
||||
echo " /* index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup */ \\"
|
||||
|
||||
ntbins=0
|
||||
nlbins=0
|
||||
lg_tiny_maxclass='"NA"'
|
||||
nbins=0
|
||||
npsizes=0
|
||||
|
||||
# Tiny size classes.
|
||||
ndelta=0
|
||||
index=0
|
||||
lg_grp=${lg_t}
|
||||
lg_delta=${lg_grp}
|
||||
while [ ${lg_grp} -lt ${lg_q} ] ; do
|
||||
size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax}
|
||||
if [ ${lg_delta_lookup} != "no" ] ; then
|
||||
nlbins=$((${index} + 1))
|
||||
fi
|
||||
if [ ${psz} = "yes" ] ; then
|
||||
npsizes=$((${npsizes} + 1))
|
||||
fi
|
||||
if [ ${bin} != "no" ] ; then
|
||||
nbins=$((${index} + 1))
|
||||
fi
|
||||
ntbins=$((${ntbins} + 1))
|
||||
lg_tiny_maxclass=${lg_grp} # Final written value is correct.
|
||||
index=$((${index} + 1))
|
||||
lg_delta=${lg_grp}
|
||||
lg_grp=$((${lg_grp} + 1))
|
||||
done
|
||||
|
||||
# First non-tiny group.
|
||||
if [ ${ntbins} -gt 0 ] ; then
|
||||
sep_line
|
||||
# The first size class has an unusual encoding, because the size has to be
|
||||
# split between grp and delta*ndelta.
|
||||
lg_grp=$((${lg_grp} - 1))
|
||||
ndelta=1
|
||||
size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax}
|
||||
index=$((${index} + 1))
|
||||
lg_grp=$((${lg_grp} + 1))
|
||||
lg_delta=$((${lg_delta} + 1))
|
||||
if [ ${psz} = "yes" ] ; then
|
||||
npsizes=$((${npsizes} + 1))
|
||||
fi
|
||||
fi
|
||||
while [ ${ndelta} -lt ${g} ] ; do
|
||||
size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax}
|
||||
index=$((${index} + 1))
|
||||
ndelta=$((${ndelta} + 1))
|
||||
if [ ${psz} = "yes" ] ; then
|
||||
npsizes=$((${npsizes} + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
# All remaining groups.
|
||||
lg_grp=$((${lg_grp} + ${lg_g}))
|
||||
while [ ${lg_grp} -lt $((${ptr_bits} - 1)) ] ; do
|
||||
sep_line
|
||||
ndelta=1
|
||||
if [ ${lg_grp} -eq $((${ptr_bits} - 2)) ] ; then
|
||||
ndelta_limit=$((${g} - 1))
|
||||
else
|
||||
ndelta_limit=${g}
|
||||
fi
|
||||
while [ ${ndelta} -le ${ndelta_limit} ] ; do
|
||||
size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax}
|
||||
if [ ${lg_delta_lookup} != "no" ] ; then
|
||||
nlbins=$((${index} + 1))
|
||||
# Final written value is correct:
|
||||
lookup_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))"
|
||||
fi
|
||||
if [ ${psz} = "yes" ] ; then
|
||||
npsizes=$((${npsizes} + 1))
|
||||
fi
|
||||
if [ ${bin} != "no" ] ; then
|
||||
nbins=$((${index} + 1))
|
||||
# Final written value is correct:
|
||||
small_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))"
|
||||
if [ ${lg_g} -gt 0 ] ; then
|
||||
lg_large_minclass=$((${lg_grp} + 1))
|
||||
else
|
||||
lg_large_minclass=$((${lg_grp} + 2))
|
||||
fi
|
||||
fi
|
||||
# Final written value is correct:
|
||||
huge_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))"
|
||||
index=$((${index} + 1))
|
||||
ndelta=$((${ndelta} + 1))
|
||||
done
|
||||
lg_grp=$((${lg_grp} + 1))
|
||||
lg_delta=$((${lg_delta} + 1))
|
||||
done
|
||||
echo
|
||||
nsizes=${index}
|
||||
|
||||
# Defined upon completion:
|
||||
# - ntbins
|
||||
# - nlbins
|
||||
# - nbins
|
||||
# - nsizes
|
||||
# - npsizes
|
||||
# - lg_tiny_maxclass
|
||||
# - lookup_maxclass
|
||||
# - small_maxclass
|
||||
# - lg_large_minclass
|
||||
# - huge_maxclass
|
||||
}
|
||||
|
||||
cat <<EOF
|
||||
/* This file was automatically generated by size_classes.sh. */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
/*
|
||||
* This header requires LG_SIZEOF_PTR, LG_TINY_MIN, LG_QUANTUM, and LG_PAGE to
|
||||
* be defined prior to inclusion, and it in turn defines:
|
||||
*
|
||||
* LG_SIZE_CLASS_GROUP: Lg of size class count for each size doubling.
|
||||
* SIZE_CLASSES: Complete table of SC(index, lg_grp, lg_delta, ndelta, psz,
|
||||
* bin, lg_delta_lookup) tuples.
|
||||
* index: Size class index.
|
||||
* lg_grp: Lg group base size (no deltas added).
|
||||
* lg_delta: Lg delta to previous size class.
|
||||
* ndelta: Delta multiplier. size == 1<<lg_grp + ndelta<<lg_delta
|
||||
* psz: 'yes' if a multiple of the page size, 'no' otherwise.
|
||||
* bin: 'yes' if a small bin size class, 'no' otherwise.
|
||||
* lg_delta_lookup: Same as lg_delta if a lookup table size class, 'no'
|
||||
* otherwise.
|
||||
* NTBINS: Number of tiny bins.
|
||||
* NLBINS: Number of bins supported by the lookup table.
|
||||
* NBINS: Number of small size class bins.
|
||||
* NSIZES: Number of size classes.
|
||||
* NPSIZES: Number of size classes that are a multiple of (1U << LG_PAGE).
|
||||
* LG_TINY_MAXCLASS: Lg of maximum tiny size class.
|
||||
* LOOKUP_MAXCLASS: Maximum size class included in lookup table.
|
||||
* SMALL_MAXCLASS: Maximum small size class.
|
||||
* LG_LARGE_MINCLASS: Lg of minimum large size class.
|
||||
* HUGE_MAXCLASS: Maximum (huge) size class.
|
||||
*/
|
||||
|
||||
#define LG_SIZE_CLASS_GROUP ${lg_g}
|
||||
|
||||
EOF
|
||||
|
||||
for lg_z in ${lg_zarr} ; do
|
||||
for lg_q in ${lg_qarr} ; do
|
||||
lg_t=${lg_tmin}
|
||||
while [ ${lg_t} -le ${lg_q} ] ; do
|
||||
# Iterate through page sizes and compute how many bins there are.
|
||||
for lg_p in ${lg_parr} ; do
|
||||
echo "#if (LG_SIZEOF_PTR == ${lg_z} && LG_TINY_MIN == ${lg_t} && LG_QUANTUM == ${lg_q} && LG_PAGE == ${lg_p})"
|
||||
size_classes ${lg_z} ${lg_q} ${lg_t} ${lg_p} ${lg_g}
|
||||
echo "#define SIZE_CLASSES_DEFINED"
|
||||
echo "#define NTBINS ${ntbins}"
|
||||
echo "#define NLBINS ${nlbins}"
|
||||
echo "#define NBINS ${nbins}"
|
||||
echo "#define NSIZES ${nsizes}"
|
||||
echo "#define NPSIZES ${npsizes}"
|
||||
echo "#define LG_TINY_MAXCLASS ${lg_tiny_maxclass}"
|
||||
echo "#define LOOKUP_MAXCLASS ${lookup_maxclass}"
|
||||
echo "#define SMALL_MAXCLASS ${small_maxclass}"
|
||||
echo "#define LG_LARGE_MINCLASS ${lg_large_minclass}"
|
||||
echo "#define HUGE_MAXCLASS ${huge_maxclass}"
|
||||
echo "#endif"
|
||||
echo
|
||||
lg_q=${lg_qmin}
|
||||
while [ ${lg_q} -le ${lg_qmax} ] ; do
|
||||
lg_t=${lg_tmin}
|
||||
while [ ${lg_t} -le ${lg_q} ] ; do
|
||||
lg_p=${lg_pmin}
|
||||
while [ ${lg_p} -le ${lg_pmax} ] ; do
|
||||
echo "#if (LG_TINY_MIN == ${lg_t} && LG_QUANTUM == ${lg_q} && LG_PAGE == ${lg_p})"
|
||||
echo "#define SIZE_CLASSES_DEFINED"
|
||||
pow2 ${lg_q}; q=${pow2_result}
|
||||
pow2 ${lg_t}; t=${pow2_result}
|
||||
pow2 ${lg_p}; p=${pow2_result}
|
||||
bin=0
|
||||
psz=0
|
||||
sz=${t}
|
||||
delta=$((${sz} - ${psz}))
|
||||
echo "/* SIZE_CLASS(bin, delta, sz) */"
|
||||
echo "#define SIZE_CLASSES \\"
|
||||
|
||||
# Tiny size classes.
|
||||
while [ ${sz} -lt ${q} ] ; do
|
||||
echo " SIZE_CLASS(${bin}, ${delta}, ${sz}) \\"
|
||||
bin=$((${bin} + 1))
|
||||
psz=${sz}
|
||||
sz=$((${sz} + ${sz}))
|
||||
delta=$((${sz} - ${psz}))
|
||||
done
|
||||
lg_t=$((${lg_t} + 1))
|
||||
# Quantum-multiple size classes. For each doubling of sz, as many as 4
|
||||
# size classes exist. Their spacing is the greater of:
|
||||
# - q
|
||||
# - sz/4, where sz is a power of 2
|
||||
while [ ${sz} -lt ${p} ] ; do
|
||||
if [ ${sz} -ge $((${q} * 4)) ] ; then
|
||||
i=$((${sz} / 4))
|
||||
else
|
||||
i=${q}
|
||||
fi
|
||||
next_2pow=$((${sz} * 2))
|
||||
while [ ${sz} -lt $next_2pow ] ; do
|
||||
echo " SIZE_CLASS(${bin}, ${delta}, ${sz}) \\"
|
||||
bin=$((${bin} + 1))
|
||||
psz=${sz}
|
||||
sz=$((${sz} + ${i}))
|
||||
delta=$((${sz} - ${psz}))
|
||||
done
|
||||
done
|
||||
echo
|
||||
echo "#define NBINS ${bin}"
|
||||
echo "#define SMALL_MAXCLASS ${psz}"
|
||||
echo "#endif"
|
||||
echo
|
||||
lg_p=$((${lg_p} + 1))
|
||||
done
|
||||
lg_t=$((${lg_t} + 1))
|
||||
done
|
||||
lg_q=$((${lg_q} + 1))
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
@@ -289,10 +92,11 @@ cat <<EOF
|
||||
#endif
|
||||
#undef SIZE_CLASSES_DEFINED
|
||||
/*
|
||||
* The size2index_tab lookup table uses uint8_t to encode each bin index, so we
|
||||
* The small_size2bin lookup table uses uint8_t to encode each bin index, so we
|
||||
* cannot support more than 256 small size classes. Further constrain NBINS to
|
||||
* 255 since all small size classes, plus a "not small" size class must be
|
||||
* stored in 8 bits of arena_chunk_map_bits_t's bits field.
|
||||
* 255 to support prof_promote, since all small size classes, plus a "not
|
||||
* small" size class must be stored in 8 bits of arena_chunk_map_t's bits
|
||||
* field.
|
||||
*/
|
||||
#if (NBINS > 255)
|
||||
# error "Too many small size classes"
|
||||
|
||||
@@ -1,246 +0,0 @@
|
||||
/*
|
||||
* This file was generated by the following command:
|
||||
* sh smoothstep.sh smoother 200 24 3 15
|
||||
*/
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
/*
|
||||
* This header defines a precomputed table based on the smoothstep family of
|
||||
* sigmoidal curves (https://en.wikipedia.org/wiki/Smoothstep) that grow from 0
|
||||
* to 1 in 0 <= x <= 1. The table is stored as integer fixed point values so
|
||||
* that floating point math can be avoided.
|
||||
*
|
||||
* 3 2
|
||||
* smoothstep(x) = -2x + 3x
|
||||
*
|
||||
* 5 4 3
|
||||
* smootherstep(x) = 6x - 15x + 10x
|
||||
*
|
||||
* 7 6 5 4
|
||||
* smootheststep(x) = -20x + 70x - 84x + 35x
|
||||
*/
|
||||
|
||||
#define SMOOTHSTEP_VARIANT "smoother"
|
||||
#define SMOOTHSTEP_NSTEPS 200
|
||||
#define SMOOTHSTEP_BFP 24
|
||||
#define SMOOTHSTEP \
|
||||
/* STEP(step, h, x, y) */ \
|
||||
STEP( 1, UINT64_C(0x0000000000000014), 0.005, 0.000001240643750) \
|
||||
STEP( 2, UINT64_C(0x00000000000000a5), 0.010, 0.000009850600000) \
|
||||
STEP( 3, UINT64_C(0x0000000000000229), 0.015, 0.000032995181250) \
|
||||
STEP( 4, UINT64_C(0x0000000000000516), 0.020, 0.000077619200000) \
|
||||
STEP( 5, UINT64_C(0x00000000000009dc), 0.025, 0.000150449218750) \
|
||||
STEP( 6, UINT64_C(0x00000000000010e8), 0.030, 0.000257995800000) \
|
||||
STEP( 7, UINT64_C(0x0000000000001aa4), 0.035, 0.000406555756250) \
|
||||
STEP( 8, UINT64_C(0x0000000000002777), 0.040, 0.000602214400000) \
|
||||
STEP( 9, UINT64_C(0x00000000000037c2), 0.045, 0.000850847793750) \
|
||||
STEP( 10, UINT64_C(0x0000000000004be6), 0.050, 0.001158125000000) \
|
||||
STEP( 11, UINT64_C(0x000000000000643c), 0.055, 0.001529510331250) \
|
||||
STEP( 12, UINT64_C(0x000000000000811f), 0.060, 0.001970265600000) \
|
||||
STEP( 13, UINT64_C(0x000000000000a2e2), 0.065, 0.002485452368750) \
|
||||
STEP( 14, UINT64_C(0x000000000000c9d8), 0.070, 0.003079934200000) \
|
||||
STEP( 15, UINT64_C(0x000000000000f64f), 0.075, 0.003758378906250) \
|
||||
STEP( 16, UINT64_C(0x0000000000012891), 0.080, 0.004525260800000) \
|
||||
STEP( 17, UINT64_C(0x00000000000160e7), 0.085, 0.005384862943750) \
|
||||
STEP( 18, UINT64_C(0x0000000000019f95), 0.090, 0.006341279400000) \
|
||||
STEP( 19, UINT64_C(0x000000000001e4dc), 0.095, 0.007398417481250) \
|
||||
STEP( 20, UINT64_C(0x00000000000230fc), 0.100, 0.008560000000000) \
|
||||
STEP( 21, UINT64_C(0x0000000000028430), 0.105, 0.009829567518750) \
|
||||
STEP( 22, UINT64_C(0x000000000002deb0), 0.110, 0.011210480600000) \
|
||||
STEP( 23, UINT64_C(0x00000000000340b1), 0.115, 0.012705922056250) \
|
||||
STEP( 24, UINT64_C(0x000000000003aa67), 0.120, 0.014318899200000) \
|
||||
STEP( 25, UINT64_C(0x0000000000041c00), 0.125, 0.016052246093750) \
|
||||
STEP( 26, UINT64_C(0x00000000000495a8), 0.130, 0.017908625800000) \
|
||||
STEP( 27, UINT64_C(0x000000000005178b), 0.135, 0.019890532631250) \
|
||||
STEP( 28, UINT64_C(0x000000000005a1cf), 0.140, 0.022000294400000) \
|
||||
STEP( 29, UINT64_C(0x0000000000063498), 0.145, 0.024240074668750) \
|
||||
STEP( 30, UINT64_C(0x000000000006d009), 0.150, 0.026611875000000) \
|
||||
STEP( 31, UINT64_C(0x000000000007743f), 0.155, 0.029117537206250) \
|
||||
STEP( 32, UINT64_C(0x0000000000082157), 0.160, 0.031758745600000) \
|
||||
STEP( 33, UINT64_C(0x000000000008d76b), 0.165, 0.034537029243750) \
|
||||
STEP( 34, UINT64_C(0x0000000000099691), 0.170, 0.037453764200000) \
|
||||
STEP( 35, UINT64_C(0x00000000000a5edf), 0.175, 0.040510175781250) \
|
||||
STEP( 36, UINT64_C(0x00000000000b3067), 0.180, 0.043707340800000) \
|
||||
STEP( 37, UINT64_C(0x00000000000c0b38), 0.185, 0.047046189818750) \
|
||||
STEP( 38, UINT64_C(0x00000000000cef5e), 0.190, 0.050527509400000) \
|
||||
STEP( 39, UINT64_C(0x00000000000ddce6), 0.195, 0.054151944356250) \
|
||||
STEP( 40, UINT64_C(0x00000000000ed3d8), 0.200, 0.057920000000000) \
|
||||
STEP( 41, UINT64_C(0x00000000000fd439), 0.205, 0.061832044393750) \
|
||||
STEP( 42, UINT64_C(0x000000000010de0e), 0.210, 0.065888310600000) \
|
||||
STEP( 43, UINT64_C(0x000000000011f158), 0.215, 0.070088898931250) \
|
||||
STEP( 44, UINT64_C(0x0000000000130e17), 0.220, 0.074433779200000) \
|
||||
STEP( 45, UINT64_C(0x0000000000143448), 0.225, 0.078922792968750) \
|
||||
STEP( 46, UINT64_C(0x00000000001563e7), 0.230, 0.083555655800000) \
|
||||
STEP( 47, UINT64_C(0x0000000000169cec), 0.235, 0.088331959506250) \
|
||||
STEP( 48, UINT64_C(0x000000000017df4f), 0.240, 0.093251174400000) \
|
||||
STEP( 49, UINT64_C(0x0000000000192b04), 0.245, 0.098312651543750) \
|
||||
STEP( 50, UINT64_C(0x00000000001a8000), 0.250, 0.103515625000000) \
|
||||
STEP( 51, UINT64_C(0x00000000001bde32), 0.255, 0.108859214081250) \
|
||||
STEP( 52, UINT64_C(0x00000000001d458b), 0.260, 0.114342425600000) \
|
||||
STEP( 53, UINT64_C(0x00000000001eb5f8), 0.265, 0.119964156118750) \
|
||||
STEP( 54, UINT64_C(0x0000000000202f65), 0.270, 0.125723194200000) \
|
||||
STEP( 55, UINT64_C(0x000000000021b1bb), 0.275, 0.131618222656250) \
|
||||
STEP( 56, UINT64_C(0x0000000000233ce3), 0.280, 0.137647820800000) \
|
||||
STEP( 57, UINT64_C(0x000000000024d0c3), 0.285, 0.143810466693750) \
|
||||
STEP( 58, UINT64_C(0x0000000000266d40), 0.290, 0.150104539400000) \
|
||||
STEP( 59, UINT64_C(0x000000000028123d), 0.295, 0.156528321231250) \
|
||||
STEP( 60, UINT64_C(0x000000000029bf9c), 0.300, 0.163080000000000) \
|
||||
STEP( 61, UINT64_C(0x00000000002b753d), 0.305, 0.169757671268750) \
|
||||
STEP( 62, UINT64_C(0x00000000002d32fe), 0.310, 0.176559340600000) \
|
||||
STEP( 63, UINT64_C(0x00000000002ef8bc), 0.315, 0.183482925806250) \
|
||||
STEP( 64, UINT64_C(0x000000000030c654), 0.320, 0.190526259200000) \
|
||||
STEP( 65, UINT64_C(0x0000000000329b9f), 0.325, 0.197687089843750) \
|
||||
STEP( 66, UINT64_C(0x0000000000347875), 0.330, 0.204963085800000) \
|
||||
STEP( 67, UINT64_C(0x0000000000365cb0), 0.335, 0.212351836381250) \
|
||||
STEP( 68, UINT64_C(0x0000000000384825), 0.340, 0.219850854400000) \
|
||||
STEP( 69, UINT64_C(0x00000000003a3aa8), 0.345, 0.227457578418750) \
|
||||
STEP( 70, UINT64_C(0x00000000003c340f), 0.350, 0.235169375000000) \
|
||||
STEP( 71, UINT64_C(0x00000000003e342b), 0.355, 0.242983540956250) \
|
||||
STEP( 72, UINT64_C(0x0000000000403ace), 0.360, 0.250897305600000) \
|
||||
STEP( 73, UINT64_C(0x00000000004247c8), 0.365, 0.258907832993750) \
|
||||
STEP( 74, UINT64_C(0x0000000000445ae9), 0.370, 0.267012224200000) \
|
||||
STEP( 75, UINT64_C(0x0000000000467400), 0.375, 0.275207519531250) \
|
||||
STEP( 76, UINT64_C(0x00000000004892d8), 0.380, 0.283490700800000) \
|
||||
STEP( 77, UINT64_C(0x00000000004ab740), 0.385, 0.291858693568750) \
|
||||
STEP( 78, UINT64_C(0x00000000004ce102), 0.390, 0.300308369400000) \
|
||||
STEP( 79, UINT64_C(0x00000000004f0fe9), 0.395, 0.308836548106250) \
|
||||
STEP( 80, UINT64_C(0x00000000005143bf), 0.400, 0.317440000000000) \
|
||||
STEP( 81, UINT64_C(0x0000000000537c4d), 0.405, 0.326115448143750) \
|
||||
STEP( 82, UINT64_C(0x000000000055b95b), 0.410, 0.334859570600000) \
|
||||
STEP( 83, UINT64_C(0x000000000057fab1), 0.415, 0.343669002681250) \
|
||||
STEP( 84, UINT64_C(0x00000000005a4015), 0.420, 0.352540339200000) \
|
||||
STEP( 85, UINT64_C(0x00000000005c894e), 0.425, 0.361470136718750) \
|
||||
STEP( 86, UINT64_C(0x00000000005ed622), 0.430, 0.370454915800000) \
|
||||
STEP( 87, UINT64_C(0x0000000000612655), 0.435, 0.379491163256250) \
|
||||
STEP( 88, UINT64_C(0x00000000006379ac), 0.440, 0.388575334400000) \
|
||||
STEP( 89, UINT64_C(0x000000000065cfeb), 0.445, 0.397703855293750) \
|
||||
STEP( 90, UINT64_C(0x00000000006828d6), 0.450, 0.406873125000000) \
|
||||
STEP( 91, UINT64_C(0x00000000006a842f), 0.455, 0.416079517831250) \
|
||||
STEP( 92, UINT64_C(0x00000000006ce1bb), 0.460, 0.425319385600000) \
|
||||
STEP( 93, UINT64_C(0x00000000006f413a), 0.465, 0.434589059868750) \
|
||||
STEP( 94, UINT64_C(0x000000000071a270), 0.470, 0.443884854200000) \
|
||||
STEP( 95, UINT64_C(0x000000000074051d), 0.475, 0.453203066406250) \
|
||||
STEP( 96, UINT64_C(0x0000000000766905), 0.480, 0.462539980800000) \
|
||||
STEP( 97, UINT64_C(0x000000000078cde7), 0.485, 0.471891870443750) \
|
||||
STEP( 98, UINT64_C(0x00000000007b3387), 0.490, 0.481254999400000) \
|
||||
STEP( 99, UINT64_C(0x00000000007d99a4), 0.495, 0.490625624981250) \
|
||||
STEP( 100, UINT64_C(0x0000000000800000), 0.500, 0.500000000000000) \
|
||||
STEP( 101, UINT64_C(0x000000000082665b), 0.505, 0.509374375018750) \
|
||||
STEP( 102, UINT64_C(0x000000000084cc78), 0.510, 0.518745000600000) \
|
||||
STEP( 103, UINT64_C(0x0000000000873218), 0.515, 0.528108129556250) \
|
||||
STEP( 104, UINT64_C(0x00000000008996fa), 0.520, 0.537460019200000) \
|
||||
STEP( 105, UINT64_C(0x00000000008bfae2), 0.525, 0.546796933593750) \
|
||||
STEP( 106, UINT64_C(0x00000000008e5d8f), 0.530, 0.556115145800000) \
|
||||
STEP( 107, UINT64_C(0x000000000090bec5), 0.535, 0.565410940131250) \
|
||||
STEP( 108, UINT64_C(0x0000000000931e44), 0.540, 0.574680614400000) \
|
||||
STEP( 109, UINT64_C(0x0000000000957bd0), 0.545, 0.583920482168750) \
|
||||
STEP( 110, UINT64_C(0x000000000097d729), 0.550, 0.593126875000000) \
|
||||
STEP( 111, UINT64_C(0x00000000009a3014), 0.555, 0.602296144706250) \
|
||||
STEP( 112, UINT64_C(0x00000000009c8653), 0.560, 0.611424665600000) \
|
||||
STEP( 113, UINT64_C(0x00000000009ed9aa), 0.565, 0.620508836743750) \
|
||||
STEP( 114, UINT64_C(0x0000000000a129dd), 0.570, 0.629545084200000) \
|
||||
STEP( 115, UINT64_C(0x0000000000a376b1), 0.575, 0.638529863281250) \
|
||||
STEP( 116, UINT64_C(0x0000000000a5bfea), 0.580, 0.647459660800000) \
|
||||
STEP( 117, UINT64_C(0x0000000000a8054e), 0.585, 0.656330997318750) \
|
||||
STEP( 118, UINT64_C(0x0000000000aa46a4), 0.590, 0.665140429400000) \
|
||||
STEP( 119, UINT64_C(0x0000000000ac83b2), 0.595, 0.673884551856250) \
|
||||
STEP( 120, UINT64_C(0x0000000000aebc40), 0.600, 0.682560000000000) \
|
||||
STEP( 121, UINT64_C(0x0000000000b0f016), 0.605, 0.691163451893750) \
|
||||
STEP( 122, UINT64_C(0x0000000000b31efd), 0.610, 0.699691630600000) \
|
||||
STEP( 123, UINT64_C(0x0000000000b548bf), 0.615, 0.708141306431250) \
|
||||
STEP( 124, UINT64_C(0x0000000000b76d27), 0.620, 0.716509299200000) \
|
||||
STEP( 125, UINT64_C(0x0000000000b98c00), 0.625, 0.724792480468750) \
|
||||
STEP( 126, UINT64_C(0x0000000000bba516), 0.630, 0.732987775800000) \
|
||||
STEP( 127, UINT64_C(0x0000000000bdb837), 0.635, 0.741092167006250) \
|
||||
STEP( 128, UINT64_C(0x0000000000bfc531), 0.640, 0.749102694400000) \
|
||||
STEP( 129, UINT64_C(0x0000000000c1cbd4), 0.645, 0.757016459043750) \
|
||||
STEP( 130, UINT64_C(0x0000000000c3cbf0), 0.650, 0.764830625000000) \
|
||||
STEP( 131, UINT64_C(0x0000000000c5c557), 0.655, 0.772542421581250) \
|
||||
STEP( 132, UINT64_C(0x0000000000c7b7da), 0.660, 0.780149145600000) \
|
||||
STEP( 133, UINT64_C(0x0000000000c9a34f), 0.665, 0.787648163618750) \
|
||||
STEP( 134, UINT64_C(0x0000000000cb878a), 0.670, 0.795036914200000) \
|
||||
STEP( 135, UINT64_C(0x0000000000cd6460), 0.675, 0.802312910156250) \
|
||||
STEP( 136, UINT64_C(0x0000000000cf39ab), 0.680, 0.809473740800000) \
|
||||
STEP( 137, UINT64_C(0x0000000000d10743), 0.685, 0.816517074193750) \
|
||||
STEP( 138, UINT64_C(0x0000000000d2cd01), 0.690, 0.823440659400000) \
|
||||
STEP( 139, UINT64_C(0x0000000000d48ac2), 0.695, 0.830242328731250) \
|
||||
STEP( 140, UINT64_C(0x0000000000d64063), 0.700, 0.836920000000000) \
|
||||
STEP( 141, UINT64_C(0x0000000000d7edc2), 0.705, 0.843471678768750) \
|
||||
STEP( 142, UINT64_C(0x0000000000d992bf), 0.710, 0.849895460600000) \
|
||||
STEP( 143, UINT64_C(0x0000000000db2f3c), 0.715, 0.856189533306250) \
|
||||
STEP( 144, UINT64_C(0x0000000000dcc31c), 0.720, 0.862352179200000) \
|
||||
STEP( 145, UINT64_C(0x0000000000de4e44), 0.725, 0.868381777343750) \
|
||||
STEP( 146, UINT64_C(0x0000000000dfd09a), 0.730, 0.874276805800000) \
|
||||
STEP( 147, UINT64_C(0x0000000000e14a07), 0.735, 0.880035843881250) \
|
||||
STEP( 148, UINT64_C(0x0000000000e2ba74), 0.740, 0.885657574400000) \
|
||||
STEP( 149, UINT64_C(0x0000000000e421cd), 0.745, 0.891140785918750) \
|
||||
STEP( 150, UINT64_C(0x0000000000e58000), 0.750, 0.896484375000000) \
|
||||
STEP( 151, UINT64_C(0x0000000000e6d4fb), 0.755, 0.901687348456250) \
|
||||
STEP( 152, UINT64_C(0x0000000000e820b0), 0.760, 0.906748825600000) \
|
||||
STEP( 153, UINT64_C(0x0000000000e96313), 0.765, 0.911668040493750) \
|
||||
STEP( 154, UINT64_C(0x0000000000ea9c18), 0.770, 0.916444344200000) \
|
||||
STEP( 155, UINT64_C(0x0000000000ebcbb7), 0.775, 0.921077207031250) \
|
||||
STEP( 156, UINT64_C(0x0000000000ecf1e8), 0.780, 0.925566220800000) \
|
||||
STEP( 157, UINT64_C(0x0000000000ee0ea7), 0.785, 0.929911101068750) \
|
||||
STEP( 158, UINT64_C(0x0000000000ef21f1), 0.790, 0.934111689400000) \
|
||||
STEP( 159, UINT64_C(0x0000000000f02bc6), 0.795, 0.938167955606250) \
|
||||
STEP( 160, UINT64_C(0x0000000000f12c27), 0.800, 0.942080000000000) \
|
||||
STEP( 161, UINT64_C(0x0000000000f22319), 0.805, 0.945848055643750) \
|
||||
STEP( 162, UINT64_C(0x0000000000f310a1), 0.810, 0.949472490600000) \
|
||||
STEP( 163, UINT64_C(0x0000000000f3f4c7), 0.815, 0.952953810181250) \
|
||||
STEP( 164, UINT64_C(0x0000000000f4cf98), 0.820, 0.956292659200000) \
|
||||
STEP( 165, UINT64_C(0x0000000000f5a120), 0.825, 0.959489824218750) \
|
||||
STEP( 166, UINT64_C(0x0000000000f6696e), 0.830, 0.962546235800000) \
|
||||
STEP( 167, UINT64_C(0x0000000000f72894), 0.835, 0.965462970756250) \
|
||||
STEP( 168, UINT64_C(0x0000000000f7dea8), 0.840, 0.968241254400000) \
|
||||
STEP( 169, UINT64_C(0x0000000000f88bc0), 0.845, 0.970882462793750) \
|
||||
STEP( 170, UINT64_C(0x0000000000f92ff6), 0.850, 0.973388125000000) \
|
||||
STEP( 171, UINT64_C(0x0000000000f9cb67), 0.855, 0.975759925331250) \
|
||||
STEP( 172, UINT64_C(0x0000000000fa5e30), 0.860, 0.977999705600000) \
|
||||
STEP( 173, UINT64_C(0x0000000000fae874), 0.865, 0.980109467368750) \
|
||||
STEP( 174, UINT64_C(0x0000000000fb6a57), 0.870, 0.982091374200000) \
|
||||
STEP( 175, UINT64_C(0x0000000000fbe400), 0.875, 0.983947753906250) \
|
||||
STEP( 176, UINT64_C(0x0000000000fc5598), 0.880, 0.985681100800000) \
|
||||
STEP( 177, UINT64_C(0x0000000000fcbf4e), 0.885, 0.987294077943750) \
|
||||
STEP( 178, UINT64_C(0x0000000000fd214f), 0.890, 0.988789519400000) \
|
||||
STEP( 179, UINT64_C(0x0000000000fd7bcf), 0.895, 0.990170432481250) \
|
||||
STEP( 180, UINT64_C(0x0000000000fdcf03), 0.900, 0.991440000000000) \
|
||||
STEP( 181, UINT64_C(0x0000000000fe1b23), 0.905, 0.992601582518750) \
|
||||
STEP( 182, UINT64_C(0x0000000000fe606a), 0.910, 0.993658720600000) \
|
||||
STEP( 183, UINT64_C(0x0000000000fe9f18), 0.915, 0.994615137056250) \
|
||||
STEP( 184, UINT64_C(0x0000000000fed76e), 0.920, 0.995474739200000) \
|
||||
STEP( 185, UINT64_C(0x0000000000ff09b0), 0.925, 0.996241621093750) \
|
||||
STEP( 186, UINT64_C(0x0000000000ff3627), 0.930, 0.996920065800000) \
|
||||
STEP( 187, UINT64_C(0x0000000000ff5d1d), 0.935, 0.997514547631250) \
|
||||
STEP( 188, UINT64_C(0x0000000000ff7ee0), 0.940, 0.998029734400000) \
|
||||
STEP( 189, UINT64_C(0x0000000000ff9bc3), 0.945, 0.998470489668750) \
|
||||
STEP( 190, UINT64_C(0x0000000000ffb419), 0.950, 0.998841875000000) \
|
||||
STEP( 191, UINT64_C(0x0000000000ffc83d), 0.955, 0.999149152206250) \
|
||||
STEP( 192, UINT64_C(0x0000000000ffd888), 0.960, 0.999397785600000) \
|
||||
STEP( 193, UINT64_C(0x0000000000ffe55b), 0.965, 0.999593444243750) \
|
||||
STEP( 194, UINT64_C(0x0000000000ffef17), 0.970, 0.999742004200000) \
|
||||
STEP( 195, UINT64_C(0x0000000000fff623), 0.975, 0.999849550781250) \
|
||||
STEP( 196, UINT64_C(0x0000000000fffae9), 0.980, 0.999922380800000) \
|
||||
STEP( 197, UINT64_C(0x0000000000fffdd6), 0.985, 0.999967004818750) \
|
||||
STEP( 198, UINT64_C(0x0000000000ffff5a), 0.990, 0.999990149400000) \
|
||||
STEP( 199, UINT64_C(0x0000000000ffffeb), 0.995, 0.999998759356250) \
|
||||
STEP( 200, UINT64_C(0x0000000001000000), 1.000, 1.000000000000000) \
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
@@ -1,115 +0,0 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Generate a discrete lookup table for a sigmoid function in the smoothstep
|
||||
# family (https://en.wikipedia.org/wiki/Smoothstep), where the lookup table
|
||||
# entries correspond to x in [1/nsteps, 2/nsteps, ..., nsteps/nsteps]. Encode
|
||||
# the entries using a binary fixed point representation.
|
||||
#
|
||||
# Usage: smoothstep.sh <variant> <nsteps> <bfp> <xprec> <yprec>
|
||||
#
|
||||
# <variant> is in {smooth, smoother, smoothest}.
|
||||
# <nsteps> must be greater than zero.
|
||||
# <bfp> must be in [0..62]; reasonable values are roughly [10..30].
|
||||
# <xprec> is x decimal precision.
|
||||
# <yprec> is y decimal precision.
|
||||
|
||||
#set -x
|
||||
|
||||
cmd="sh smoothstep.sh $*"
|
||||
variant=$1
|
||||
nsteps=$2
|
||||
bfp=$3
|
||||
xprec=$4
|
||||
yprec=$5
|
||||
|
||||
case "${variant}" in
|
||||
smooth)
|
||||
;;
|
||||
smoother)
|
||||
;;
|
||||
smoothest)
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported variant"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
smooth() {
|
||||
step=$1
|
||||
y=`echo ${yprec} k ${step} ${nsteps} / sx _2 lx 3 ^ '*' 3 lx 2 ^ '*' + p | dc | tr -d '\\\\\n' | sed -e 's#^\.#0.#g'`
|
||||
h=`echo ${yprec} k 2 ${bfp} ^ ${y} '*' p | dc | tr -d '\\\\\n' | sed -e 's#^\.#0.#g' | tr '.' ' ' | awk '{print $1}' `
|
||||
}
|
||||
|
||||
smoother() {
|
||||
step=$1
|
||||
y=`echo ${yprec} k ${step} ${nsteps} / sx 6 lx 5 ^ '*' _15 lx 4 ^ '*' + 10 lx 3 ^ '*' + p | dc | tr -d '\\\\\n' | sed -e 's#^\.#0.#g'`
|
||||
h=`echo ${yprec} k 2 ${bfp} ^ ${y} '*' p | dc | tr -d '\\\\\n' | sed -e 's#^\.#0.#g' | tr '.' ' ' | awk '{print $1}' `
|
||||
}
|
||||
|
||||
smoothest() {
|
||||
step=$1
|
||||
y=`echo ${yprec} k ${step} ${nsteps} / sx _20 lx 7 ^ '*' 70 lx 6 ^ '*' + _84 lx 5 ^ '*' + 35 lx 4 ^ '*' + p | dc | tr -d '\\\\\n' | sed -e 's#^\.#0.#g'`
|
||||
h=`echo ${yprec} k 2 ${bfp} ^ ${y} '*' p | dc | tr -d '\\\\\n' | sed -e 's#^\.#0.#g' | tr '.' ' ' | awk '{print $1}' `
|
||||
}
|
||||
|
||||
cat <<EOF
|
||||
/*
|
||||
* This file was generated by the following command:
|
||||
* $cmd
|
||||
*/
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
/*
|
||||
* This header defines a precomputed table based on the smoothstep family of
|
||||
* sigmoidal curves (https://en.wikipedia.org/wiki/Smoothstep) that grow from 0
|
||||
* to 1 in 0 <= x <= 1. The table is stored as integer fixed point values so
|
||||
* that floating point math can be avoided.
|
||||
*
|
||||
* 3 2
|
||||
* smoothstep(x) = -2x + 3x
|
||||
*
|
||||
* 5 4 3
|
||||
* smootherstep(x) = 6x - 15x + 10x
|
||||
*
|
||||
* 7 6 5 4
|
||||
* smootheststep(x) = -20x + 70x - 84x + 35x
|
||||
*/
|
||||
|
||||
#define SMOOTHSTEP_VARIANT "${variant}"
|
||||
#define SMOOTHSTEP_NSTEPS ${nsteps}
|
||||
#define SMOOTHSTEP_BFP ${bfp}
|
||||
#define SMOOTHSTEP \\
|
||||
/* STEP(step, h, x, y) */ \\
|
||||
EOF
|
||||
|
||||
s=1
|
||||
while [ $s -le $nsteps ] ; do
|
||||
$variant ${s}
|
||||
x=`echo ${xprec} k ${s} ${nsteps} / p | dc | tr -d '\\\\\n' | sed -e 's#^\.#0.#g'`
|
||||
printf ' STEP(%4d, UINT64_C(0x%016x), %s, %s) \\\n' ${s} ${h} ${x} ${y}
|
||||
|
||||
s=$((s+1))
|
||||
done
|
||||
echo
|
||||
|
||||
cat <<EOF
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
EOF
|
||||
@@ -1,51 +0,0 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
typedef struct spin_s spin_t;
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
|
||||
struct spin_s {
|
||||
unsigned iteration;
|
||||
};
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
void spin_init(spin_t *spin);
|
||||
void spin_adaptive(spin_t *spin);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_SPIN_C_))
|
||||
JEMALLOC_INLINE void
|
||||
spin_init(spin_t *spin)
|
||||
{
|
||||
|
||||
spin->iteration = 0;
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
spin_adaptive(spin_t *spin)
|
||||
{
|
||||
volatile uint64_t i;
|
||||
|
||||
for (i = 0; i < (KQU(1) << spin->iteration); i++)
|
||||
CPU_SPINWAIT;
|
||||
|
||||
if (spin->iteration < 63)
|
||||
spin->iteration++;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
typedef struct tcache_bin_stats_s tcache_bin_stats_t;
|
||||
typedef struct malloc_bin_stats_s malloc_bin_stats_t;
|
||||
typedef struct malloc_large_stats_s malloc_large_stats_t;
|
||||
typedef struct malloc_huge_stats_s malloc_huge_stats_t;
|
||||
typedef struct arena_stats_s arena_stats_t;
|
||||
typedef struct chunk_stats_s chunk_stats_t;
|
||||
|
||||
@@ -21,6 +20,12 @@ struct tcache_bin_stats_s {
|
||||
};
|
||||
|
||||
struct malloc_bin_stats_s {
|
||||
/*
|
||||
* Current number of bytes allocated, including objects currently
|
||||
* cached by tcache.
|
||||
*/
|
||||
size_t allocated;
|
||||
|
||||
/*
|
||||
* Total number of allocation/deallocation requests served directly by
|
||||
* the bin. Note that tcache may allocate an object, then recycle it
|
||||
@@ -37,12 +42,6 @@ struct malloc_bin_stats_s {
|
||||
*/
|
||||
uint64_t nrequests;
|
||||
|
||||
/*
|
||||
* Current number of regions of this size class, including regions
|
||||
* currently cached by tcache.
|
||||
*/
|
||||
size_t curregs;
|
||||
|
||||
/* Number of tcache fills from this bin. */
|
||||
uint64_t nfills;
|
||||
|
||||
@@ -79,37 +78,14 @@ struct malloc_large_stats_s {
|
||||
*/
|
||||
uint64_t nrequests;
|
||||
|
||||
/*
|
||||
* Current number of runs of this size class, including runs currently
|
||||
* cached by tcache.
|
||||
*/
|
||||
/* Current number of runs of this size class. */
|
||||
size_t curruns;
|
||||
};
|
||||
|
||||
struct malloc_huge_stats_s {
|
||||
/*
|
||||
* Total number of allocation/deallocation requests served directly by
|
||||
* the arena.
|
||||
*/
|
||||
uint64_t nmalloc;
|
||||
uint64_t ndalloc;
|
||||
|
||||
/* Current number of (multi-)chunk allocations of this size class. */
|
||||
size_t curhchunks;
|
||||
};
|
||||
|
||||
struct arena_stats_s {
|
||||
/* Number of bytes currently mapped. */
|
||||
size_t mapped;
|
||||
|
||||
/*
|
||||
* Number of bytes currently retained as a side effect of munmap() being
|
||||
* disabled/bypassed. Retained bytes are technically mapped (though
|
||||
* always decommitted or purged), but they are excluded from the mapped
|
||||
* statistic (above).
|
||||
*/
|
||||
size_t retained;
|
||||
|
||||
/*
|
||||
* Total number of purge sweeps, total number of madvise calls made,
|
||||
* and total pages purged in order to keep dirty unused memory under
|
||||
@@ -119,28 +95,34 @@ struct arena_stats_s {
|
||||
uint64_t nmadvise;
|
||||
uint64_t purged;
|
||||
|
||||
/*
|
||||
* Number of bytes currently mapped purely for metadata purposes, and
|
||||
* number of bytes currently allocated for internal metadata.
|
||||
*/
|
||||
size_t metadata_mapped;
|
||||
size_t metadata_allocated; /* Protected via atomic_*_z(). */
|
||||
|
||||
/* Per-size-category statistics. */
|
||||
size_t allocated_large;
|
||||
uint64_t nmalloc_large;
|
||||
uint64_t ndalloc_large;
|
||||
uint64_t nrequests_large;
|
||||
|
||||
size_t allocated_huge;
|
||||
uint64_t nmalloc_huge;
|
||||
uint64_t ndalloc_huge;
|
||||
|
||||
/* One element for each large size class. */
|
||||
/*
|
||||
* One element for each possible size class, including sizes that
|
||||
* overlap with bin size classes. This is necessary because ipalloc()
|
||||
* sometimes has to use such large objects in order to assure proper
|
||||
* alignment.
|
||||
*/
|
||||
malloc_large_stats_t *lstats;
|
||||
};
|
||||
|
||||
/* One element for each huge size class. */
|
||||
malloc_huge_stats_t *hstats;
|
||||
struct chunk_stats_s {
|
||||
/* Number of chunks that were allocated. */
|
||||
uint64_t nchunks;
|
||||
|
||||
/* High-water mark for number of chunks allocated. */
|
||||
size_t highchunks;
|
||||
|
||||
/*
|
||||
* Current number of chunks allocated. This value isn't maintained for
|
||||
* any other purpose, so keep track of it in order to be able to set
|
||||
* highchunks.
|
||||
*/
|
||||
size_t curchunks;
|
||||
};
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
@@ -176,9 +158,6 @@ JEMALLOC_INLINE void
|
||||
stats_cactive_add(size_t size)
|
||||
{
|
||||
|
||||
assert(size > 0);
|
||||
assert((size & chunksize_mask) == 0);
|
||||
|
||||
atomic_add_z(&stats_cactive, size);
|
||||
}
|
||||
|
||||
@@ -186,9 +165,6 @@ JEMALLOC_INLINE void
|
||||
stats_cactive_sub(size_t size)
|
||||
{
|
||||
|
||||
assert(size > 0);
|
||||
assert((size & chunksize_mask) == 0);
|
||||
|
||||
atomic_sub_z(&stats_cactive, size);
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
typedef struct tcache_bin_info_s tcache_bin_info_t;
|
||||
typedef struct tcache_bin_s tcache_bin_t;
|
||||
typedef struct tcache_s tcache_t;
|
||||
typedef struct tcaches_s tcaches_t;
|
||||
|
||||
/*
|
||||
* tcache pointers close to NULL are used to encode state information that is
|
||||
@@ -16,11 +15,6 @@ typedef struct tcaches_s tcaches_t;
|
||||
#define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3)
|
||||
#define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY
|
||||
|
||||
/*
|
||||
* Absolute minimum number of cache slots for each small bin.
|
||||
*/
|
||||
#define TCACHE_NSLOTS_SMALL_MIN 20
|
||||
|
||||
/*
|
||||
* Absolute maximum number of cache slots for each small bin in the thread
|
||||
* cache. This is an additional constraint beyond that imposed as: twice the
|
||||
@@ -70,21 +64,15 @@ struct tcache_bin_s {
|
||||
int low_water; /* Min # cached since last GC. */
|
||||
unsigned lg_fill_div; /* Fill (ncached_max >> lg_fill_div). */
|
||||
unsigned ncached; /* # of cached objects. */
|
||||
/*
|
||||
* To make use of adjacent cacheline prefetch, the items in the avail
|
||||
* stack goes to higher address for newer allocations. avail points
|
||||
* just above the available space, which means that
|
||||
* avail[-ncached, ... -1] are available items and the lowest item will
|
||||
* be allocated first.
|
||||
*/
|
||||
void **avail; /* Stack of available objects. */
|
||||
};
|
||||
|
||||
struct tcache_s {
|
||||
ql_elm(tcache_t) link; /* Used for aggregating stats. */
|
||||
uint64_t prof_accumbytes;/* Cleared after arena_prof_accum(). */
|
||||
ticker_t gc_ticker; /* Drives incremental GC. */
|
||||
szind_t next_gc_bin; /* Next bin to GC. */
|
||||
uint64_t prof_accumbytes;/* Cleared after arena_prof_accum() */
|
||||
arena_t *arena; /* This thread's arena. */
|
||||
unsigned ev_cnt; /* Event count since incremental GC. */
|
||||
unsigned next_gc_bin; /* Next bin to GC. */
|
||||
tcache_bin_t tbins[1]; /* Dynamically sized. */
|
||||
/*
|
||||
* The pointer stacks associated with tbins follow as a contiguous
|
||||
@@ -94,14 +82,6 @@ struct tcache_s {
|
||||
*/
|
||||
};
|
||||
|
||||
/* Linkage for list of available (previously used) explicit tcache IDs. */
|
||||
struct tcaches_s {
|
||||
union {
|
||||
tcache_t *tcache;
|
||||
tcaches_t *next;
|
||||
};
|
||||
};
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
@@ -115,88 +95,84 @@ extern tcache_bin_info_t *tcache_bin_info;
|
||||
* Number of tcache bins. There are NBINS small-object bins, plus 0 or more
|
||||
* large-object bins.
|
||||
*/
|
||||
extern unsigned nhbins;
|
||||
extern size_t nhbins;
|
||||
|
||||
/* Maximum cached size class. */
|
||||
extern size_t tcache_maxclass;
|
||||
extern size_t tcache_maxclass;
|
||||
|
||||
/*
|
||||
* Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
|
||||
* usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are
|
||||
* completely disjoint from this data structure. tcaches starts off as a sparse
|
||||
* array, so it has no physical memory footprint until individual pages are
|
||||
* touched. This allows the entire array to be allocated the first time an
|
||||
* explicit tcache is created without a disproportionate impact on memory usage.
|
||||
*/
|
||||
extern tcaches_t *tcaches;
|
||||
|
||||
size_t tcache_salloc(tsdn_t *tsdn, const void *ptr);
|
||||
void tcache_event_hard(tsd_t *tsd, tcache_t *tcache);
|
||||
void *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
|
||||
tcache_bin_t *tbin, szind_t binind, bool *tcache_success);
|
||||
void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
|
||||
szind_t binind, unsigned rem);
|
||||
void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
|
||||
unsigned rem, tcache_t *tcache);
|
||||
void tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache,
|
||||
arena_t *oldarena, arena_t *newarena);
|
||||
tcache_t *tcache_get_hard(tsd_t *tsd);
|
||||
tcache_t *tcache_create(tsdn_t *tsdn, arena_t *arena);
|
||||
void tcache_cleanup(tsd_t *tsd);
|
||||
void tcache_enabled_cleanup(tsd_t *tsd);
|
||||
void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
|
||||
bool tcaches_create(tsd_t *tsd, unsigned *r_ind);
|
||||
void tcaches_flush(tsd_t *tsd, unsigned ind);
|
||||
void tcaches_destroy(tsd_t *tsd, unsigned ind);
|
||||
bool tcache_boot(tsdn_t *tsdn);
|
||||
size_t tcache_salloc(const void *ptr);
|
||||
void tcache_event_hard(tcache_t *tcache);
|
||||
void *tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin,
|
||||
size_t binind);
|
||||
void tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem,
|
||||
tcache_t *tcache);
|
||||
void tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
|
||||
tcache_t *tcache);
|
||||
void tcache_arena_associate(tcache_t *tcache, arena_t *arena);
|
||||
void tcache_arena_dissociate(tcache_t *tcache);
|
||||
tcache_t *tcache_create(arena_t *arena);
|
||||
void tcache_destroy(tcache_t *tcache);
|
||||
void tcache_thread_cleanup(void *arg);
|
||||
void tcache_stats_merge(tcache_t *tcache, arena_t *arena);
|
||||
bool tcache_boot0(void);
|
||||
bool tcache_boot1(void);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
void tcache_event(tsd_t *tsd, tcache_t *tcache);
|
||||
malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache, tcache_t *)
|
||||
malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache_enabled, tcache_enabled_t)
|
||||
|
||||
void tcache_event(tcache_t *tcache);
|
||||
void tcache_flush(void);
|
||||
bool tcache_enabled_get(void);
|
||||
tcache_t *tcache_get(tsd_t *tsd, bool create);
|
||||
tcache_t *tcache_get(bool create);
|
||||
void tcache_enabled_set(bool enabled);
|
||||
void *tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success);
|
||||
void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
|
||||
size_t size, szind_t ind, bool zero, bool slow_path);
|
||||
void *tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
|
||||
size_t size, szind_t ind, bool zero, bool slow_path);
|
||||
void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr,
|
||||
szind_t binind, bool slow_path);
|
||||
void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr,
|
||||
size_t size, bool slow_path);
|
||||
tcache_t *tcaches_get(tsd_t *tsd, unsigned ind);
|
||||
void *tcache_alloc_easy(tcache_bin_t *tbin);
|
||||
void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero);
|
||||
void *tcache_alloc_large(tcache_t *tcache, size_t size, bool zero);
|
||||
void tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind);
|
||||
void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
|
||||
/* Map of thread-specific caches. */
|
||||
malloc_tsd_externs(tcache, tcache_t *)
|
||||
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache, tcache_t *, NULL,
|
||||
tcache_thread_cleanup)
|
||||
/* Per thread flag that allows thread caches to be disabled. */
|
||||
malloc_tsd_externs(tcache_enabled, tcache_enabled_t)
|
||||
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache_enabled, tcache_enabled_t,
|
||||
tcache_enabled_default, malloc_tsd_no_cleanup)
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
tcache_flush(void)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
tcache_t *tcache;
|
||||
|
||||
cassert(config_tcache);
|
||||
|
||||
tsd = tsd_fetch();
|
||||
tcache_cleanup(tsd);
|
||||
tcache = *tcache_tsd_get();
|
||||
if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)
|
||||
return;
|
||||
tcache_destroy(tcache);
|
||||
tcache = NULL;
|
||||
tcache_tsd_set(&tcache);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
tcache_enabled_get(void)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
tcache_enabled_t tcache_enabled;
|
||||
|
||||
cassert(config_tcache);
|
||||
|
||||
tsd = tsd_fetch();
|
||||
tcache_enabled = tsd_tcache_enabled_get(tsd);
|
||||
tcache_enabled = *tcache_enabled_tsd_get();
|
||||
if (tcache_enabled == tcache_enabled_default) {
|
||||
tcache_enabled = (tcache_enabled_t)opt_tcache;
|
||||
tsd_tcache_enabled_set(tsd, tcache_enabled);
|
||||
tcache_enabled_tsd_set(&tcache_enabled);
|
||||
}
|
||||
|
||||
return ((bool)tcache_enabled);
|
||||
@@ -205,171 +181,180 @@ tcache_enabled_get(void)
|
||||
JEMALLOC_INLINE void
|
||||
tcache_enabled_set(bool enabled)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
tcache_enabled_t tcache_enabled;
|
||||
tcache_t *tcache;
|
||||
|
||||
cassert(config_tcache);
|
||||
|
||||
tsd = tsd_fetch();
|
||||
|
||||
tcache_enabled = (tcache_enabled_t)enabled;
|
||||
tsd_tcache_enabled_set(tsd, tcache_enabled);
|
||||
|
||||
if (!enabled)
|
||||
tcache_cleanup(tsd);
|
||||
tcache_enabled_tsd_set(&tcache_enabled);
|
||||
tcache = *tcache_tsd_get();
|
||||
if (enabled) {
|
||||
if (tcache == TCACHE_STATE_DISABLED) {
|
||||
tcache = NULL;
|
||||
tcache_tsd_set(&tcache);
|
||||
}
|
||||
} else /* disabled */ {
|
||||
if (tcache > TCACHE_STATE_MAX) {
|
||||
tcache_destroy(tcache);
|
||||
tcache = NULL;
|
||||
}
|
||||
if (tcache == NULL) {
|
||||
tcache = TCACHE_STATE_DISABLED;
|
||||
tcache_tsd_set(&tcache);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE tcache_t *
|
||||
tcache_get(tsd_t *tsd, bool create)
|
||||
tcache_get(bool create)
|
||||
{
|
||||
tcache_t *tcache;
|
||||
|
||||
if (!config_tcache)
|
||||
if (config_tcache == false)
|
||||
return (NULL);
|
||||
if (config_lazy_lock && isthreaded == false)
|
||||
return (NULL);
|
||||
|
||||
tcache = tsd_tcache_get(tsd);
|
||||
if (!create)
|
||||
return (tcache);
|
||||
if (unlikely(tcache == NULL) && tsd_nominal(tsd)) {
|
||||
tcache = tcache_get_hard(tsd);
|
||||
tsd_tcache_set(tsd, tcache);
|
||||
tcache = *tcache_tsd_get();
|
||||
if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX) {
|
||||
if (tcache == TCACHE_STATE_DISABLED)
|
||||
return (NULL);
|
||||
if (tcache == NULL) {
|
||||
if (create == false) {
|
||||
/*
|
||||
* Creating a tcache here would cause
|
||||
* allocation as a side effect of free().
|
||||
* Ordinarily that would be okay since
|
||||
* tcache_create() failure is a soft failure
|
||||
* that doesn't propagate. However, if TLS
|
||||
* data are freed via free() as in glibc,
|
||||
* subtle corruption could result from setting
|
||||
* a TLS variable after its backing memory is
|
||||
* freed.
|
||||
*/
|
||||
return (NULL);
|
||||
}
|
||||
if (tcache_enabled_get() == false) {
|
||||
tcache_enabled_set(false); /* Memoize. */
|
||||
return (NULL);
|
||||
}
|
||||
return (tcache_create(choose_arena(NULL)));
|
||||
}
|
||||
if (tcache == TCACHE_STATE_PURGATORY) {
|
||||
/*
|
||||
* Make a note that an allocator function was called
|
||||
* after tcache_thread_cleanup() was called.
|
||||
*/
|
||||
tcache = TCACHE_STATE_REINCARNATED;
|
||||
tcache_tsd_set(&tcache);
|
||||
return (NULL);
|
||||
}
|
||||
if (tcache == TCACHE_STATE_REINCARNATED)
|
||||
return (NULL);
|
||||
not_reached();
|
||||
}
|
||||
|
||||
return (tcache);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
tcache_event(tsd_t *tsd, tcache_t *tcache)
|
||||
tcache_event(tcache_t *tcache)
|
||||
{
|
||||
|
||||
if (TCACHE_GC_INCR == 0)
|
||||
return;
|
||||
|
||||
if (unlikely(ticker_tick(&tcache->gc_ticker)))
|
||||
tcache_event_hard(tsd, tcache);
|
||||
tcache->ev_cnt++;
|
||||
assert(tcache->ev_cnt <= TCACHE_GC_INCR);
|
||||
if (tcache->ev_cnt == TCACHE_GC_INCR)
|
||||
tcache_event_hard(tcache);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void *
|
||||
tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success)
|
||||
tcache_alloc_easy(tcache_bin_t *tbin)
|
||||
{
|
||||
void *ret;
|
||||
|
||||
if (unlikely(tbin->ncached == 0)) {
|
||||
if (tbin->ncached == 0) {
|
||||
tbin->low_water = -1;
|
||||
*tcache_success = false;
|
||||
return (NULL);
|
||||
}
|
||||
/*
|
||||
* tcache_success (instead of ret) should be checked upon the return of
|
||||
* this function. We avoid checking (ret == NULL) because there is
|
||||
* never a null stored on the avail stack (which is unknown to the
|
||||
* compiler), and eagerly checking ret would cause pipeline stall
|
||||
* (waiting for the cacheline).
|
||||
*/
|
||||
*tcache_success = true;
|
||||
ret = *(tbin->avail - tbin->ncached);
|
||||
tbin->ncached--;
|
||||
|
||||
if (unlikely((int)tbin->ncached < tbin->low_water))
|
||||
if ((int)tbin->ncached < tbin->low_water)
|
||||
tbin->low_water = tbin->ncached;
|
||||
|
||||
ret = tbin->avail[tbin->ncached];
|
||||
return (ret);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void *
|
||||
tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
|
||||
szind_t binind, bool zero, bool slow_path)
|
||||
tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
|
||||
{
|
||||
void *ret;
|
||||
size_t binind;
|
||||
tcache_bin_t *tbin;
|
||||
bool tcache_success;
|
||||
size_t usize JEMALLOC_CC_SILENCE_INIT(0);
|
||||
|
||||
binind = SMALL_SIZE2BIN(size);
|
||||
assert(binind < NBINS);
|
||||
tbin = &tcache->tbins[binind];
|
||||
ret = tcache_alloc_easy(tbin, &tcache_success);
|
||||
assert(tcache_success == (ret != NULL));
|
||||
if (unlikely(!tcache_success)) {
|
||||
bool tcache_hard_success;
|
||||
arena = arena_choose(tsd, arena);
|
||||
if (unlikely(arena == NULL))
|
||||
return (NULL);
|
||||
|
||||
ret = tcache_alloc_small_hard(tsd_tsdn(tsd), arena, tcache,
|
||||
tbin, binind, &tcache_hard_success);
|
||||
if (tcache_hard_success == false)
|
||||
size = arena_bin_info[binind].reg_size;
|
||||
ret = tcache_alloc_easy(tbin);
|
||||
if (ret == NULL) {
|
||||
ret = tcache_alloc_small_hard(tcache, tbin, binind);
|
||||
if (ret == NULL)
|
||||
return (NULL);
|
||||
}
|
||||
assert(tcache_salloc(ret) == arena_bin_info[binind].reg_size);
|
||||
|
||||
assert(ret);
|
||||
/*
|
||||
* Only compute usize if required. The checks in the following if
|
||||
* statement are all static.
|
||||
*/
|
||||
if (config_prof || (slow_path && config_fill) || unlikely(zero)) {
|
||||
usize = index2size(binind);
|
||||
assert(tcache_salloc(tsd_tsdn(tsd), ret) == usize);
|
||||
}
|
||||
|
||||
if (likely(!zero)) {
|
||||
if (slow_path && config_fill) {
|
||||
if (unlikely(opt_junk_alloc)) {
|
||||
if (zero == false) {
|
||||
if (config_fill) {
|
||||
if (opt_junk) {
|
||||
arena_alloc_junk_small(ret,
|
||||
&arena_bin_info[binind], false);
|
||||
} else if (unlikely(opt_zero))
|
||||
memset(ret, 0, usize);
|
||||
} else if (opt_zero)
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
|
||||
} else {
|
||||
if (slow_path && config_fill && unlikely(opt_junk_alloc)) {
|
||||
if (config_fill && opt_junk) {
|
||||
arena_alloc_junk_small(ret, &arena_bin_info[binind],
|
||||
true);
|
||||
}
|
||||
memset(ret, 0, usize);
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
|
||||
if (config_stats)
|
||||
tbin->tstats.nrequests++;
|
||||
if (config_prof)
|
||||
tcache->prof_accumbytes += usize;
|
||||
tcache_event(tsd, tcache);
|
||||
tcache->prof_accumbytes += arena_bin_info[binind].reg_size;
|
||||
tcache_event(tcache);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void *
|
||||
tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
|
||||
szind_t binind, bool zero, bool slow_path)
|
||||
tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
|
||||
{
|
||||
void *ret;
|
||||
size_t binind;
|
||||
tcache_bin_t *tbin;
|
||||
bool tcache_success;
|
||||
|
||||
size = PAGE_CEILING(size);
|
||||
assert(size <= tcache_maxclass);
|
||||
binind = NBINS + (size >> LG_PAGE) - 1;
|
||||
assert(binind < nhbins);
|
||||
tbin = &tcache->tbins[binind];
|
||||
ret = tcache_alloc_easy(tbin, &tcache_success);
|
||||
assert(tcache_success == (ret != NULL));
|
||||
if (unlikely(!tcache_success)) {
|
||||
ret = tcache_alloc_easy(tbin);
|
||||
if (ret == NULL) {
|
||||
/*
|
||||
* Only allocate one large object at a time, because it's quite
|
||||
* expensive to create one and not use it.
|
||||
*/
|
||||
arena = arena_choose(tsd, arena);
|
||||
if (unlikely(arena == NULL))
|
||||
return (NULL);
|
||||
|
||||
ret = arena_malloc_large(tsd_tsdn(tsd), arena, binind, zero);
|
||||
ret = arena_malloc_large(tcache->arena, size, zero);
|
||||
if (ret == NULL)
|
||||
return (NULL);
|
||||
} else {
|
||||
size_t usize JEMALLOC_CC_SILENCE_INIT(0);
|
||||
|
||||
/* Only compute usize on demand */
|
||||
if (config_prof || (slow_path && config_fill) ||
|
||||
unlikely(zero)) {
|
||||
usize = index2size(binind);
|
||||
assert(usize <= tcache_maxclass);
|
||||
}
|
||||
|
||||
if (config_prof && usize == LARGE_MINCLASS) {
|
||||
if (config_prof && prof_promote && size == PAGE) {
|
||||
arena_chunk_t *chunk =
|
||||
(arena_chunk_t *)CHUNK_ADDR2BASE(ret);
|
||||
size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >>
|
||||
@@ -377,91 +362,80 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
|
||||
arena_mapbits_large_binind_set(chunk, pageind,
|
||||
BININD_INVALID);
|
||||
}
|
||||
if (likely(!zero)) {
|
||||
if (slow_path && config_fill) {
|
||||
if (unlikely(opt_junk_alloc)) {
|
||||
memset(ret, JEMALLOC_ALLOC_JUNK,
|
||||
usize);
|
||||
} else if (unlikely(opt_zero))
|
||||
memset(ret, 0, usize);
|
||||
if (zero == false) {
|
||||
if (config_fill) {
|
||||
if (opt_junk)
|
||||
memset(ret, 0xa5, size);
|
||||
else if (opt_zero)
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
} else
|
||||
memset(ret, 0, usize);
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
|
||||
} else {
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
|
||||
if (config_stats)
|
||||
tbin->tstats.nrequests++;
|
||||
if (config_prof)
|
||||
tcache->prof_accumbytes += usize;
|
||||
tcache->prof_accumbytes += size;
|
||||
}
|
||||
|
||||
tcache_event(tsd, tcache);
|
||||
tcache_event(tcache);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
|
||||
bool slow_path)
|
||||
tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind)
|
||||
{
|
||||
tcache_bin_t *tbin;
|
||||
tcache_bin_info_t *tbin_info;
|
||||
|
||||
assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= SMALL_MAXCLASS);
|
||||
assert(tcache_salloc(ptr) <= SMALL_MAXCLASS);
|
||||
|
||||
if (slow_path && config_fill && unlikely(opt_junk_free))
|
||||
if (config_fill && opt_junk)
|
||||
arena_dalloc_junk_small(ptr, &arena_bin_info[binind]);
|
||||
|
||||
tbin = &tcache->tbins[binind];
|
||||
tbin_info = &tcache_bin_info[binind];
|
||||
if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
|
||||
tcache_bin_flush_small(tsd, tcache, tbin, binind,
|
||||
(tbin_info->ncached_max >> 1));
|
||||
if (tbin->ncached == tbin_info->ncached_max) {
|
||||
tcache_bin_flush_small(tbin, binind, (tbin_info->ncached_max >>
|
||||
1), tcache);
|
||||
}
|
||||
assert(tbin->ncached < tbin_info->ncached_max);
|
||||
tbin->avail[tbin->ncached] = ptr;
|
||||
tbin->ncached++;
|
||||
*(tbin->avail - tbin->ncached) = ptr;
|
||||
|
||||
tcache_event(tsd, tcache);
|
||||
tcache_event(tcache);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size,
|
||||
bool slow_path)
|
||||
tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size)
|
||||
{
|
||||
szind_t binind;
|
||||
size_t binind;
|
||||
tcache_bin_t *tbin;
|
||||
tcache_bin_info_t *tbin_info;
|
||||
|
||||
assert((size & PAGE_MASK) == 0);
|
||||
assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SMALL_MAXCLASS);
|
||||
assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= tcache_maxclass);
|
||||
assert(tcache_salloc(ptr) > SMALL_MAXCLASS);
|
||||
assert(tcache_salloc(ptr) <= tcache_maxclass);
|
||||
|
||||
binind = size2index(size);
|
||||
binind = NBINS + (size >> LG_PAGE) - 1;
|
||||
|
||||
if (slow_path && config_fill && unlikely(opt_junk_free))
|
||||
arena_dalloc_junk_large(ptr, size);
|
||||
if (config_fill && opt_junk)
|
||||
memset(ptr, 0x5a, size);
|
||||
|
||||
tbin = &tcache->tbins[binind];
|
||||
tbin_info = &tcache_bin_info[binind];
|
||||
if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
|
||||
tcache_bin_flush_large(tsd, tbin, binind,
|
||||
(tbin_info->ncached_max >> 1), tcache);
|
||||
if (tbin->ncached == tbin_info->ncached_max) {
|
||||
tcache_bin_flush_large(tbin, binind, (tbin_info->ncached_max >>
|
||||
1), tcache);
|
||||
}
|
||||
assert(tbin->ncached < tbin_info->ncached_max);
|
||||
tbin->avail[tbin->ncached] = ptr;
|
||||
tbin->ncached++;
|
||||
*(tbin->avail - tbin->ncached) = ptr;
|
||||
|
||||
tcache_event(tsd, tcache);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE tcache_t *
|
||||
tcaches_get(tsd_t *tsd, unsigned ind)
|
||||
{
|
||||
tcaches_t *elm = &tcaches[ind];
|
||||
if (unlikely(elm->tcache == NULL)) {
|
||||
elm->tcache = tcache_create(tsd_tsdn(tsd), arena_choose(tsd,
|
||||
NULL));
|
||||
}
|
||||
return (elm->tcache);
|
||||
tcache_event(tcache);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
typedef struct ticker_s ticker_t;
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
|
||||
struct ticker_s {
|
||||
int32_t tick;
|
||||
int32_t nticks;
|
||||
};
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
void ticker_init(ticker_t *ticker, int32_t nticks);
|
||||
void ticker_copy(ticker_t *ticker, const ticker_t *other);
|
||||
int32_t ticker_read(const ticker_t *ticker);
|
||||
bool ticker_ticks(ticker_t *ticker, int32_t nticks);
|
||||
bool ticker_tick(ticker_t *ticker);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TICKER_C_))
|
||||
JEMALLOC_INLINE void
|
||||
ticker_init(ticker_t *ticker, int32_t nticks)
|
||||
{
|
||||
|
||||
ticker->tick = nticks;
|
||||
ticker->nticks = nticks;
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
ticker_copy(ticker_t *ticker, const ticker_t *other)
|
||||
{
|
||||
|
||||
*ticker = *other;
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE int32_t
|
||||
ticker_read(const ticker_t *ticker)
|
||||
{
|
||||
|
||||
return (ticker->tick);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
ticker_ticks(ticker_t *ticker, int32_t nticks)
|
||||
{
|
||||
|
||||
if (unlikely(ticker->tick < nticks)) {
|
||||
ticker->tick = ticker->nticks;
|
||||
return (true);
|
||||
}
|
||||
ticker->tick -= nticks;
|
||||
return(false);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
ticker_tick(ticker_t *ticker)
|
||||
{
|
||||
|
||||
return (ticker_ticks(ticker, 1));
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
@@ -2,7 +2,7 @@
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
/* Maximum number of malloc_tsd users with cleanup functions. */
|
||||
#define MALLOC_TSD_CLEANUPS_MAX 2
|
||||
#define MALLOC_TSD_CLEANUPS_MAX 8
|
||||
|
||||
typedef bool (*malloc_tsd_cleanup_t)(void);
|
||||
|
||||
@@ -12,21 +12,9 @@ typedef struct tsd_init_block_s tsd_init_block_t;
|
||||
typedef struct tsd_init_head_s tsd_init_head_t;
|
||||
#endif
|
||||
|
||||
typedef struct tsd_s tsd_t;
|
||||
typedef struct tsdn_s tsdn_t;
|
||||
|
||||
#define TSDN_NULL ((tsdn_t *)0)
|
||||
|
||||
typedef enum {
|
||||
tsd_state_uninitialized,
|
||||
tsd_state_nominal,
|
||||
tsd_state_purgatory,
|
||||
tsd_state_reincarnated
|
||||
} tsd_state_t;
|
||||
|
||||
/*
|
||||
* TLS/TSD-agnostic macro-based implementation of thread-specific data. There
|
||||
* are five macros that support (at least) three use cases: file-private,
|
||||
* are four macros that support (at least) three use cases: file-private,
|
||||
* library-private, and library-private inlined. Following is an example
|
||||
* library-private tsd variable:
|
||||
*
|
||||
@@ -36,37 +24,34 @@ typedef enum {
|
||||
* int y;
|
||||
* } example_t;
|
||||
* #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
|
||||
* malloc_tsd_types(example_, example_t)
|
||||
* malloc_tsd_protos(, example_, example_t)
|
||||
* malloc_tsd_externs(example_, example_t)
|
||||
* malloc_tsd_protos(, example, example_t *)
|
||||
* malloc_tsd_externs(example, example_t *)
|
||||
* In example.c:
|
||||
* malloc_tsd_data(, example_, example_t, EX_INITIALIZER)
|
||||
* malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER,
|
||||
* malloc_tsd_data(, example, example_t *, EX_INITIALIZER)
|
||||
* malloc_tsd_funcs(, example, example_t *, EX_INITIALIZER,
|
||||
* example_tsd_cleanup)
|
||||
*
|
||||
* The result is a set of generated functions, e.g.:
|
||||
*
|
||||
* bool example_tsd_boot(void) {...}
|
||||
* bool example_tsd_booted_get(void) {...}
|
||||
* example_t *example_tsd_get(bool init) {...}
|
||||
* void example_tsd_set(example_t *val) {...}
|
||||
* example_t **example_tsd_get() {...}
|
||||
* void example_tsd_set(example_t **val) {...}
|
||||
*
|
||||
* Note that all of the functions deal in terms of (a_type *) rather than
|
||||
* (a_type) so that it is possible to support non-pointer types (unlike
|
||||
* (a_type) so that it is possible to support non-pointer types (unlike
|
||||
* pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is
|
||||
* cast to (void *). This means that the cleanup function needs to cast the
|
||||
* function argument to (a_type *), then dereference the resulting pointer to
|
||||
* access fields, e.g.
|
||||
* cast to (void *). This means that the cleanup function needs to cast *and*
|
||||
* dereference the function argument, e.g.:
|
||||
*
|
||||
* void
|
||||
* example_tsd_cleanup(void *arg)
|
||||
* {
|
||||
* example_t *example = (example_t *)arg;
|
||||
* example_t *example = *(example_t **)arg;
|
||||
*
|
||||
* example->x = 42;
|
||||
* [...]
|
||||
* if ([want the cleanup function to be called again])
|
||||
* example_tsd_set(example);
|
||||
* if ([want the cleanup function to be called again]) {
|
||||
* example_tsd_set(&example);
|
||||
* }
|
||||
* }
|
||||
*
|
||||
* If example_tsd_set() is called within example_tsd_cleanup(), it will be
|
||||
@@ -75,98 +60,63 @@ typedef enum {
|
||||
* non-NULL.
|
||||
*/
|
||||
|
||||
/* malloc_tsd_types(). */
|
||||
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
|
||||
#define malloc_tsd_types(a_name, a_type)
|
||||
#elif (defined(JEMALLOC_TLS))
|
||||
#define malloc_tsd_types(a_name, a_type)
|
||||
#elif (defined(_WIN32))
|
||||
#define malloc_tsd_types(a_name, a_type) \
|
||||
typedef struct { \
|
||||
bool initialized; \
|
||||
a_type val; \
|
||||
} a_name##tsd_wrapper_t;
|
||||
#else
|
||||
#define malloc_tsd_types(a_name, a_type) \
|
||||
typedef struct { \
|
||||
bool initialized; \
|
||||
a_type val; \
|
||||
} a_name##tsd_wrapper_t;
|
||||
#endif
|
||||
|
||||
/* malloc_tsd_protos(). */
|
||||
#define malloc_tsd_protos(a_attr, a_name, a_type) \
|
||||
a_attr bool \
|
||||
a_name##tsd_boot0(void); \
|
||||
a_attr void \
|
||||
a_name##tsd_boot1(void); \
|
||||
a_attr bool \
|
||||
a_name##tsd_boot(void); \
|
||||
a_attr bool \
|
||||
a_name##tsd_booted_get(void); \
|
||||
a_name##_tsd_boot(void); \
|
||||
a_attr a_type * \
|
||||
a_name##tsd_get(bool init); \
|
||||
a_name##_tsd_get(void); \
|
||||
a_attr void \
|
||||
a_name##tsd_set(a_type *val);
|
||||
a_name##_tsd_set(a_type *val);
|
||||
|
||||
/* malloc_tsd_externs(). */
|
||||
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
|
||||
#define malloc_tsd_externs(a_name, a_type) \
|
||||
extern __thread a_type a_name##tsd_tls; \
|
||||
extern __thread bool a_name##tsd_initialized; \
|
||||
extern bool a_name##tsd_booted;
|
||||
extern __thread a_type a_name##_tls; \
|
||||
extern __thread bool a_name##_initialized; \
|
||||
extern bool a_name##_booted;
|
||||
#elif (defined(JEMALLOC_TLS))
|
||||
#define malloc_tsd_externs(a_name, a_type) \
|
||||
extern __thread a_type a_name##tsd_tls; \
|
||||
extern pthread_key_t a_name##tsd_tsd; \
|
||||
extern bool a_name##tsd_booted;
|
||||
extern __thread a_type a_name##_tls; \
|
||||
extern pthread_key_t a_name##_tsd; \
|
||||
extern bool a_name##_booted;
|
||||
#elif (defined(_WIN32))
|
||||
#define malloc_tsd_externs(a_name, a_type) \
|
||||
extern DWORD a_name##tsd_tsd; \
|
||||
extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \
|
||||
extern bool a_name##tsd_booted;
|
||||
extern DWORD a_name##_tsd; \
|
||||
extern bool a_name##_booted;
|
||||
#else
|
||||
#define malloc_tsd_externs(a_name, a_type) \
|
||||
extern pthread_key_t a_name##tsd_tsd; \
|
||||
extern tsd_init_head_t a_name##tsd_init_head; \
|
||||
extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \
|
||||
extern bool a_name##tsd_booted;
|
||||
extern pthread_key_t a_name##_tsd; \
|
||||
extern tsd_init_head_t a_name##_tsd_init_head; \
|
||||
extern bool a_name##_booted;
|
||||
#endif
|
||||
|
||||
/* malloc_tsd_data(). */
|
||||
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
|
||||
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
|
||||
a_attr __thread a_type JEMALLOC_TLS_MODEL \
|
||||
a_name##tsd_tls = a_initializer; \
|
||||
a_name##_tls = a_initializer; \
|
||||
a_attr __thread bool JEMALLOC_TLS_MODEL \
|
||||
a_name##tsd_initialized = false; \
|
||||
a_attr bool a_name##tsd_booted = false;
|
||||
a_name##_initialized = false; \
|
||||
a_attr bool a_name##_booted = false;
|
||||
#elif (defined(JEMALLOC_TLS))
|
||||
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
|
||||
a_attr __thread a_type JEMALLOC_TLS_MODEL \
|
||||
a_name##tsd_tls = a_initializer; \
|
||||
a_attr pthread_key_t a_name##tsd_tsd; \
|
||||
a_attr bool a_name##tsd_booted = false;
|
||||
a_name##_tls = a_initializer; \
|
||||
a_attr pthread_key_t a_name##_tsd; \
|
||||
a_attr bool a_name##_booted = false;
|
||||
#elif (defined(_WIN32))
|
||||
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
|
||||
a_attr DWORD a_name##tsd_tsd; \
|
||||
a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \
|
||||
false, \
|
||||
a_initializer \
|
||||
}; \
|
||||
a_attr bool a_name##tsd_booted = false;
|
||||
a_attr DWORD a_name##_tsd; \
|
||||
a_attr bool a_name##_booted = false;
|
||||
#else
|
||||
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
|
||||
a_attr pthread_key_t a_name##tsd_tsd; \
|
||||
a_attr tsd_init_head_t a_name##tsd_init_head = { \
|
||||
a_attr pthread_key_t a_name##_tsd; \
|
||||
a_attr tsd_init_head_t a_name##_tsd_init_head = { \
|
||||
ql_head_initializer(blocks), \
|
||||
MALLOC_MUTEX_INITIALIZER \
|
||||
}; \
|
||||
a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \
|
||||
false, \
|
||||
a_initializer \
|
||||
}; \
|
||||
a_attr bool a_name##tsd_booted = false;
|
||||
a_attr bool a_name##_booted = false;
|
||||
#endif
|
||||
|
||||
/* malloc_tsd_funcs(). */
|
||||
@@ -175,124 +125,75 @@ a_attr bool a_name##tsd_booted = false;
|
||||
a_cleanup) \
|
||||
/* Initialization/cleanup. */ \
|
||||
a_attr bool \
|
||||
a_name##tsd_cleanup_wrapper(void) \
|
||||
a_name##_tsd_cleanup_wrapper(void) \
|
||||
{ \
|
||||
\
|
||||
if (a_name##tsd_initialized) { \
|
||||
a_name##tsd_initialized = false; \
|
||||
a_cleanup(&a_name##tsd_tls); \
|
||||
if (a_name##_initialized) { \
|
||||
a_name##_initialized = false; \
|
||||
a_cleanup(&a_name##_tls); \
|
||||
} \
|
||||
return (a_name##tsd_initialized); \
|
||||
return (a_name##_initialized); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_boot0(void) \
|
||||
a_name##_tsd_boot(void) \
|
||||
{ \
|
||||
\
|
||||
if (a_cleanup != malloc_tsd_no_cleanup) { \
|
||||
malloc_tsd_cleanup_register( \
|
||||
&a_name##tsd_cleanup_wrapper); \
|
||||
&a_name##_tsd_cleanup_wrapper); \
|
||||
} \
|
||||
a_name##tsd_booted = true; \
|
||||
return (false); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_name##tsd_boot1(void) \
|
||||
{ \
|
||||
\
|
||||
/* Do nothing. */ \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_boot(void) \
|
||||
{ \
|
||||
\
|
||||
return (a_name##tsd_boot0()); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_booted_get(void) \
|
||||
{ \
|
||||
\
|
||||
return (a_name##tsd_booted); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_get_allocates(void) \
|
||||
{ \
|
||||
\
|
||||
a_name##_booted = true; \
|
||||
return (false); \
|
||||
} \
|
||||
/* Get/set. */ \
|
||||
a_attr a_type * \
|
||||
a_name##tsd_get(bool init) \
|
||||
a_name##_tsd_get(void) \
|
||||
{ \
|
||||
\
|
||||
assert(a_name##tsd_booted); \
|
||||
return (&a_name##tsd_tls); \
|
||||
assert(a_name##_booted); \
|
||||
return (&a_name##_tls); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_name##tsd_set(a_type *val) \
|
||||
a_name##_tsd_set(a_type *val) \
|
||||
{ \
|
||||
\
|
||||
assert(a_name##tsd_booted); \
|
||||
a_name##tsd_tls = (*val); \
|
||||
assert(a_name##_booted); \
|
||||
a_name##_tls = (*val); \
|
||||
if (a_cleanup != malloc_tsd_no_cleanup) \
|
||||
a_name##tsd_initialized = true; \
|
||||
a_name##_initialized = true; \
|
||||
}
|
||||
#elif (defined(JEMALLOC_TLS))
|
||||
#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
|
||||
a_cleanup) \
|
||||
/* Initialization/cleanup. */ \
|
||||
a_attr bool \
|
||||
a_name##tsd_boot0(void) \
|
||||
a_name##_tsd_boot(void) \
|
||||
{ \
|
||||
\
|
||||
if (a_cleanup != malloc_tsd_no_cleanup) { \
|
||||
if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \
|
||||
0) \
|
||||
if (pthread_key_create(&a_name##_tsd, a_cleanup) != 0) \
|
||||
return (true); \
|
||||
} \
|
||||
a_name##tsd_booted = true; \
|
||||
return (false); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_name##tsd_boot1(void) \
|
||||
{ \
|
||||
\
|
||||
/* Do nothing. */ \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_boot(void) \
|
||||
{ \
|
||||
\
|
||||
return (a_name##tsd_boot0()); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_booted_get(void) \
|
||||
{ \
|
||||
\
|
||||
return (a_name##tsd_booted); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_get_allocates(void) \
|
||||
{ \
|
||||
\
|
||||
a_name##_booted = true; \
|
||||
return (false); \
|
||||
} \
|
||||
/* Get/set. */ \
|
||||
a_attr a_type * \
|
||||
a_name##tsd_get(bool init) \
|
||||
a_name##_tsd_get(void) \
|
||||
{ \
|
||||
\
|
||||
assert(a_name##tsd_booted); \
|
||||
return (&a_name##tsd_tls); \
|
||||
assert(a_name##_booted); \
|
||||
return (&a_name##_tls); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_name##tsd_set(a_type *val) \
|
||||
a_name##_tsd_set(a_type *val) \
|
||||
{ \
|
||||
\
|
||||
assert(a_name##tsd_booted); \
|
||||
a_name##tsd_tls = (*val); \
|
||||
assert(a_name##_booted); \
|
||||
a_name##_tls = (*val); \
|
||||
if (a_cleanup != malloc_tsd_no_cleanup) { \
|
||||
if (pthread_setspecific(a_name##tsd_tsd, \
|
||||
(void *)(&a_name##tsd_tls))) { \
|
||||
if (pthread_setspecific(a_name##_tsd, \
|
||||
(void *)(&a_name##_tls))) { \
|
||||
malloc_write("<jemalloc>: Error" \
|
||||
" setting TSD for "#a_name"\n"); \
|
||||
if (opt_abort) \
|
||||
@@ -303,21 +204,27 @@ a_name##tsd_set(a_type *val) \
|
||||
#elif (defined(_WIN32))
|
||||
#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
|
||||
a_cleanup) \
|
||||
/* Data structure. */ \
|
||||
typedef struct { \
|
||||
bool initialized; \
|
||||
a_type val; \
|
||||
} a_name##_tsd_wrapper_t; \
|
||||
/* Initialization/cleanup. */ \
|
||||
a_attr bool \
|
||||
a_name##tsd_cleanup_wrapper(void) \
|
||||
a_name##_tsd_cleanup_wrapper(void) \
|
||||
{ \
|
||||
DWORD error = GetLastError(); \
|
||||
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
|
||||
TlsGetValue(a_name##tsd_tsd); \
|
||||
SetLastError(error); \
|
||||
a_name##_tsd_wrapper_t *wrapper; \
|
||||
\
|
||||
wrapper = (a_name##_tsd_wrapper_t *) TlsGetValue(a_name##_tsd); \
|
||||
if (wrapper == NULL) \
|
||||
return (false); \
|
||||
if (a_cleanup != malloc_tsd_no_cleanup && \
|
||||
wrapper->initialized) { \
|
||||
a_type val = wrapper->val; \
|
||||
a_type tsd_static_data = a_initializer; \
|
||||
wrapper->initialized = false; \
|
||||
a_cleanup(&wrapper->val); \
|
||||
wrapper->val = tsd_static_data; \
|
||||
a_cleanup(&val); \
|
||||
if (wrapper->initialized) { \
|
||||
/* Trigger another cleanup round. */ \
|
||||
return (true); \
|
||||
@@ -326,109 +233,63 @@ a_name##tsd_cleanup_wrapper(void) \
|
||||
malloc_tsd_dalloc(wrapper); \
|
||||
return (false); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \
|
||||
a_attr bool \
|
||||
a_name##_tsd_boot(void) \
|
||||
{ \
|
||||
\
|
||||
if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \
|
||||
malloc_write("<jemalloc>: Error setting" \
|
||||
" TSD for "#a_name"\n"); \
|
||||
abort(); \
|
||||
a_name##_tsd = TlsAlloc(); \
|
||||
if (a_name##_tsd == TLS_OUT_OF_INDEXES) \
|
||||
return (true); \
|
||||
if (a_cleanup != malloc_tsd_no_cleanup) { \
|
||||
malloc_tsd_cleanup_register( \
|
||||
&a_name##_tsd_cleanup_wrapper); \
|
||||
} \
|
||||
a_name##_booted = true; \
|
||||
return (false); \
|
||||
} \
|
||||
a_attr a_name##tsd_wrapper_t * \
|
||||
a_name##tsd_wrapper_get(bool init) \
|
||||
/* Get/set. */ \
|
||||
a_attr a_name##_tsd_wrapper_t * \
|
||||
a_name##_tsd_get_wrapper(void) \
|
||||
{ \
|
||||
DWORD error = GetLastError(); \
|
||||
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
|
||||
TlsGetValue(a_name##tsd_tsd); \
|
||||
SetLastError(error); \
|
||||
a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \
|
||||
TlsGetValue(a_name##_tsd); \
|
||||
\
|
||||
if (init && unlikely(wrapper == NULL)) { \
|
||||
wrapper = (a_name##tsd_wrapper_t *) \
|
||||
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
|
||||
if (wrapper == NULL) { \
|
||||
wrapper = (a_name##_tsd_wrapper_t *) \
|
||||
malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \
|
||||
if (wrapper == NULL) { \
|
||||
malloc_write("<jemalloc>: Error allocating" \
|
||||
" TSD for "#a_name"\n"); \
|
||||
abort(); \
|
||||
} else { \
|
||||
static a_type tsd_static_data = a_initializer; \
|
||||
wrapper->initialized = false; \
|
||||
wrapper->val = a_initializer; \
|
||||
wrapper->val = tsd_static_data; \
|
||||
} \
|
||||
if (!TlsSetValue(a_name##_tsd, (void *)wrapper)) { \
|
||||
malloc_write("<jemalloc>: Error setting" \
|
||||
" TSD for "#a_name"\n"); \
|
||||
abort(); \
|
||||
} \
|
||||
a_name##tsd_wrapper_set(wrapper); \
|
||||
} \
|
||||
return (wrapper); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_boot0(void) \
|
||||
{ \
|
||||
\
|
||||
a_name##tsd_tsd = TlsAlloc(); \
|
||||
if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \
|
||||
return (true); \
|
||||
if (a_cleanup != malloc_tsd_no_cleanup) { \
|
||||
malloc_tsd_cleanup_register( \
|
||||
&a_name##tsd_cleanup_wrapper); \
|
||||
} \
|
||||
a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \
|
||||
a_name##tsd_booted = true; \
|
||||
return (false); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_name##tsd_boot1(void) \
|
||||
{ \
|
||||
a_name##tsd_wrapper_t *wrapper; \
|
||||
wrapper = (a_name##tsd_wrapper_t *) \
|
||||
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
|
||||
if (wrapper == NULL) { \
|
||||
malloc_write("<jemalloc>: Error allocating" \
|
||||
" TSD for "#a_name"\n"); \
|
||||
abort(); \
|
||||
} \
|
||||
memcpy(wrapper, &a_name##tsd_boot_wrapper, \
|
||||
sizeof(a_name##tsd_wrapper_t)); \
|
||||
a_name##tsd_wrapper_set(wrapper); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_boot(void) \
|
||||
{ \
|
||||
\
|
||||
if (a_name##tsd_boot0()) \
|
||||
return (true); \
|
||||
a_name##tsd_boot1(); \
|
||||
return (false); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_booted_get(void) \
|
||||
{ \
|
||||
\
|
||||
return (a_name##tsd_booted); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_get_allocates(void) \
|
||||
{ \
|
||||
\
|
||||
return (true); \
|
||||
} \
|
||||
/* Get/set. */ \
|
||||
a_attr a_type * \
|
||||
a_name##tsd_get(bool init) \
|
||||
a_name##_tsd_get(void) \
|
||||
{ \
|
||||
a_name##tsd_wrapper_t *wrapper; \
|
||||
a_name##_tsd_wrapper_t *wrapper; \
|
||||
\
|
||||
assert(a_name##tsd_booted); \
|
||||
wrapper = a_name##tsd_wrapper_get(init); \
|
||||
if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \
|
||||
return (NULL); \
|
||||
assert(a_name##_booted); \
|
||||
wrapper = a_name##_tsd_get_wrapper(); \
|
||||
return (&wrapper->val); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_name##tsd_set(a_type *val) \
|
||||
a_name##_tsd_set(a_type *val) \
|
||||
{ \
|
||||
a_name##tsd_wrapper_t *wrapper; \
|
||||
a_name##_tsd_wrapper_t *wrapper; \
|
||||
\
|
||||
assert(a_name##tsd_booted); \
|
||||
wrapper = a_name##tsd_wrapper_get(true); \
|
||||
assert(a_name##_booted); \
|
||||
wrapper = a_name##_tsd_get_wrapper(); \
|
||||
wrapper->val = *(val); \
|
||||
if (a_cleanup != malloc_tsd_no_cleanup) \
|
||||
wrapper->initialized = true; \
|
||||
@@ -436,11 +297,16 @@ a_name##tsd_set(a_type *val) \
|
||||
#else
|
||||
#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
|
||||
a_cleanup) \
|
||||
/* Data structure. */ \
|
||||
typedef struct { \
|
||||
bool initialized; \
|
||||
a_type val; \
|
||||
} a_name##_tsd_wrapper_t; \
|
||||
/* Initialization/cleanup. */ \
|
||||
a_attr void \
|
||||
a_name##tsd_cleanup_wrapper(void *arg) \
|
||||
a_name##_tsd_cleanup_wrapper(void *arg) \
|
||||
{ \
|
||||
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \
|
||||
a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)arg;\
|
||||
\
|
||||
if (a_cleanup != malloc_tsd_no_cleanup && \
|
||||
wrapper->initialized) { \
|
||||
@@ -448,7 +314,7 @@ a_name##tsd_cleanup_wrapper(void *arg) \
|
||||
a_cleanup(&wrapper->val); \
|
||||
if (wrapper->initialized) { \
|
||||
/* Trigger another cleanup round. */ \
|
||||
if (pthread_setspecific(a_name##tsd_tsd, \
|
||||
if (pthread_setspecific(a_name##_tsd, \
|
||||
(void *)wrapper)) { \
|
||||
malloc_write("<jemalloc>: Error" \
|
||||
" setting TSD for "#a_name"\n"); \
|
||||
@@ -460,111 +326,67 @@ a_name##tsd_cleanup_wrapper(void *arg) \
|
||||
} \
|
||||
malloc_tsd_dalloc(wrapper); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \
|
||||
a_attr bool \
|
||||
a_name##_tsd_boot(void) \
|
||||
{ \
|
||||
\
|
||||
if (pthread_setspecific(a_name##tsd_tsd, \
|
||||
(void *)wrapper)) { \
|
||||
malloc_write("<jemalloc>: Error setting" \
|
||||
" TSD for "#a_name"\n"); \
|
||||
abort(); \
|
||||
} \
|
||||
if (pthread_key_create(&a_name##_tsd, \
|
||||
a_name##_tsd_cleanup_wrapper) != 0) \
|
||||
return (true); \
|
||||
a_name##_booted = true; \
|
||||
return (false); \
|
||||
} \
|
||||
a_attr a_name##tsd_wrapper_t * \
|
||||
a_name##tsd_wrapper_get(bool init) \
|
||||
/* Get/set. */ \
|
||||
a_attr a_name##_tsd_wrapper_t * \
|
||||
a_name##_tsd_get_wrapper(void) \
|
||||
{ \
|
||||
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
|
||||
pthread_getspecific(a_name##tsd_tsd); \
|
||||
a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \
|
||||
pthread_getspecific(a_name##_tsd); \
|
||||
\
|
||||
if (init && unlikely(wrapper == NULL)) { \
|
||||
if (wrapper == NULL) { \
|
||||
tsd_init_block_t block; \
|
||||
wrapper = tsd_init_check_recursion( \
|
||||
&a_name##tsd_init_head, &block); \
|
||||
&a_name##_tsd_init_head, &block); \
|
||||
if (wrapper) \
|
||||
return (wrapper); \
|
||||
wrapper = (a_name##tsd_wrapper_t *) \
|
||||
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
|
||||
wrapper = (a_name##_tsd_wrapper_t *) \
|
||||
malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \
|
||||
block.data = wrapper; \
|
||||
if (wrapper == NULL) { \
|
||||
malloc_write("<jemalloc>: Error allocating" \
|
||||
" TSD for "#a_name"\n"); \
|
||||
abort(); \
|
||||
} else { \
|
||||
static a_type tsd_static_data = a_initializer; \
|
||||
wrapper->initialized = false; \
|
||||
wrapper->val = a_initializer; \
|
||||
wrapper->val = tsd_static_data; \
|
||||
} \
|
||||
a_name##tsd_wrapper_set(wrapper); \
|
||||
tsd_init_finish(&a_name##tsd_init_head, &block); \
|
||||
if (pthread_setspecific(a_name##_tsd, \
|
||||
(void *)wrapper)) { \
|
||||
malloc_write("<jemalloc>: Error setting" \
|
||||
" TSD for "#a_name"\n"); \
|
||||
abort(); \
|
||||
} \
|
||||
tsd_init_finish(&a_name##_tsd_init_head, &block); \
|
||||
} \
|
||||
return (wrapper); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_boot0(void) \
|
||||
{ \
|
||||
\
|
||||
if (pthread_key_create(&a_name##tsd_tsd, \
|
||||
a_name##tsd_cleanup_wrapper) != 0) \
|
||||
return (true); \
|
||||
a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \
|
||||
a_name##tsd_booted = true; \
|
||||
return (false); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_name##tsd_boot1(void) \
|
||||
{ \
|
||||
a_name##tsd_wrapper_t *wrapper; \
|
||||
wrapper = (a_name##tsd_wrapper_t *) \
|
||||
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
|
||||
if (wrapper == NULL) { \
|
||||
malloc_write("<jemalloc>: Error allocating" \
|
||||
" TSD for "#a_name"\n"); \
|
||||
abort(); \
|
||||
} \
|
||||
memcpy(wrapper, &a_name##tsd_boot_wrapper, \
|
||||
sizeof(a_name##tsd_wrapper_t)); \
|
||||
a_name##tsd_wrapper_set(wrapper); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_boot(void) \
|
||||
{ \
|
||||
\
|
||||
if (a_name##tsd_boot0()) \
|
||||
return (true); \
|
||||
a_name##tsd_boot1(); \
|
||||
return (false); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_booted_get(void) \
|
||||
{ \
|
||||
\
|
||||
return (a_name##tsd_booted); \
|
||||
} \
|
||||
a_attr bool \
|
||||
a_name##tsd_get_allocates(void) \
|
||||
{ \
|
||||
\
|
||||
return (true); \
|
||||
} \
|
||||
/* Get/set. */ \
|
||||
a_attr a_type * \
|
||||
a_name##tsd_get(bool init) \
|
||||
a_name##_tsd_get(void) \
|
||||
{ \
|
||||
a_name##tsd_wrapper_t *wrapper; \
|
||||
a_name##_tsd_wrapper_t *wrapper; \
|
||||
\
|
||||
assert(a_name##tsd_booted); \
|
||||
wrapper = a_name##tsd_wrapper_get(init); \
|
||||
if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \
|
||||
return (NULL); \
|
||||
assert(a_name##_booted); \
|
||||
wrapper = a_name##_tsd_get_wrapper(); \
|
||||
return (&wrapper->val); \
|
||||
} \
|
||||
a_attr void \
|
||||
a_name##tsd_set(a_type *val) \
|
||||
a_name##_tsd_set(a_type *val) \
|
||||
{ \
|
||||
a_name##tsd_wrapper_t *wrapper; \
|
||||
a_name##_tsd_wrapper_t *wrapper; \
|
||||
\
|
||||
assert(a_name##tsd_booted); \
|
||||
wrapper = a_name##tsd_wrapper_get(true); \
|
||||
assert(a_name##_booted); \
|
||||
wrapper = a_name##_tsd_get_wrapper(); \
|
||||
wrapper->val = *(val); \
|
||||
if (a_cleanup != malloc_tsd_no_cleanup) \
|
||||
wrapper->initialized = true; \
|
||||
@@ -588,200 +410,25 @@ struct tsd_init_head_s {
|
||||
};
|
||||
#endif
|
||||
|
||||
#define MALLOC_TSD \
|
||||
/* O(name, type) */ \
|
||||
O(tcache, tcache_t *) \
|
||||
O(thread_allocated, uint64_t) \
|
||||
O(thread_deallocated, uint64_t) \
|
||||
O(prof_tdata, prof_tdata_t *) \
|
||||
O(iarena, arena_t *) \
|
||||
O(arena, arena_t *) \
|
||||
O(arenas_tdata, arena_tdata_t *) \
|
||||
O(narenas_tdata, unsigned) \
|
||||
O(arenas_tdata_bypass, bool) \
|
||||
O(tcache_enabled, tcache_enabled_t) \
|
||||
O(quarantine, quarantine_t *) \
|
||||
O(witnesses, witness_list_t) \
|
||||
O(witness_fork, bool) \
|
||||
|
||||
#define TSD_INITIALIZER { \
|
||||
tsd_state_uninitialized, \
|
||||
NULL, \
|
||||
0, \
|
||||
0, \
|
||||
NULL, \
|
||||
NULL, \
|
||||
NULL, \
|
||||
NULL, \
|
||||
0, \
|
||||
false, \
|
||||
tcache_enabled_default, \
|
||||
NULL, \
|
||||
ql_head_initializer(witnesses), \
|
||||
false \
|
||||
}
|
||||
|
||||
struct tsd_s {
|
||||
tsd_state_t state;
|
||||
#define O(n, t) \
|
||||
t n;
|
||||
MALLOC_TSD
|
||||
#undef O
|
||||
};
|
||||
|
||||
/*
|
||||
* Wrapper around tsd_t that makes it possible to avoid implicit conversion
|
||||
* between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be
|
||||
* explicitly converted to tsd_t, which is non-nullable.
|
||||
*/
|
||||
struct tsdn_s {
|
||||
tsd_t tsd;
|
||||
};
|
||||
|
||||
static const tsd_t tsd_initializer = TSD_INITIALIZER;
|
||||
|
||||
malloc_tsd_types(, tsd_t)
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
void *malloc_tsd_malloc(size_t size);
|
||||
void malloc_tsd_dalloc(void *wrapper);
|
||||
void malloc_tsd_no_cleanup(void *arg);
|
||||
void malloc_tsd_no_cleanup(void *);
|
||||
void malloc_tsd_cleanup_register(bool (*f)(void));
|
||||
tsd_t *malloc_tsd_boot0(void);
|
||||
void malloc_tsd_boot1(void);
|
||||
void malloc_tsd_boot(void);
|
||||
#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
|
||||
!defined(_WIN32))
|
||||
void *tsd_init_check_recursion(tsd_init_head_t *head,
|
||||
tsd_init_block_t *block);
|
||||
void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
|
||||
#endif
|
||||
void tsd_cleanup(void *arg);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
|
||||
|
||||
tsd_t *tsd_fetch_impl(bool init);
|
||||
tsd_t *tsd_fetch(void);
|
||||
tsdn_t *tsd_tsdn(tsd_t *tsd);
|
||||
bool tsd_nominal(tsd_t *tsd);
|
||||
#define O(n, t) \
|
||||
t *tsd_##n##p_get(tsd_t *tsd); \
|
||||
t tsd_##n##_get(tsd_t *tsd); \
|
||||
void tsd_##n##_set(tsd_t *tsd, t n);
|
||||
MALLOC_TSD
|
||||
#undef O
|
||||
tsdn_t *tsdn_fetch(void);
|
||||
bool tsdn_null(const tsdn_t *tsdn);
|
||||
tsd_t *tsdn_tsd(tsdn_t *tsdn);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
|
||||
malloc_tsd_externs(, tsd_t)
|
||||
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE tsd_t *
|
||||
tsd_fetch_impl(bool init)
|
||||
{
|
||||
tsd_t *tsd = tsd_get(init);
|
||||
|
||||
if (!init && tsd_get_allocates() && tsd == NULL)
|
||||
return (NULL);
|
||||
assert(tsd != NULL);
|
||||
|
||||
if (unlikely(tsd->state != tsd_state_nominal)) {
|
||||
if (tsd->state == tsd_state_uninitialized) {
|
||||
tsd->state = tsd_state_nominal;
|
||||
/* Trigger cleanup handler registration. */
|
||||
tsd_set(tsd);
|
||||
} else if (tsd->state == tsd_state_purgatory) {
|
||||
tsd->state = tsd_state_reincarnated;
|
||||
tsd_set(tsd);
|
||||
} else
|
||||
assert(tsd->state == tsd_state_reincarnated);
|
||||
}
|
||||
|
||||
return (tsd);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE tsd_t *
|
||||
tsd_fetch(void)
|
||||
{
|
||||
|
||||
return (tsd_fetch_impl(true));
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE tsdn_t *
|
||||
tsd_tsdn(tsd_t *tsd)
|
||||
{
|
||||
|
||||
return ((tsdn_t *)tsd);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE bool
|
||||
tsd_nominal(tsd_t *tsd)
|
||||
{
|
||||
|
||||
return (tsd->state == tsd_state_nominal);
|
||||
}
|
||||
|
||||
#define O(n, t) \
|
||||
JEMALLOC_ALWAYS_INLINE t * \
|
||||
tsd_##n##p_get(tsd_t *tsd) \
|
||||
{ \
|
||||
\
|
||||
return (&tsd->n); \
|
||||
} \
|
||||
\
|
||||
JEMALLOC_ALWAYS_INLINE t \
|
||||
tsd_##n##_get(tsd_t *tsd) \
|
||||
{ \
|
||||
\
|
||||
return (*tsd_##n##p_get(tsd)); \
|
||||
} \
|
||||
\
|
||||
JEMALLOC_ALWAYS_INLINE void \
|
||||
tsd_##n##_set(tsd_t *tsd, t n) \
|
||||
{ \
|
||||
\
|
||||
assert(tsd->state == tsd_state_nominal); \
|
||||
tsd->n = n; \
|
||||
}
|
||||
MALLOC_TSD
|
||||
#undef O
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE tsdn_t *
|
||||
tsdn_fetch(void)
|
||||
{
|
||||
|
||||
if (!tsd_booted_get())
|
||||
return (NULL);
|
||||
|
||||
return (tsd_tsdn(tsd_fetch_impl(false)));
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
tsdn_null(const tsdn_t *tsdn)
|
||||
{
|
||||
|
||||
return (tsdn == NULL);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE tsd_t *
|
||||
tsdn_tsd(tsdn_t *tsdn)
|
||||
{
|
||||
|
||||
assert(!tsdn_null(tsdn));
|
||||
|
||||
return (&tsdn->tsd);
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
|
||||
@@ -1,36 +1,6 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
#ifdef _WIN32
|
||||
# ifdef _WIN64
|
||||
# define FMT64_PREFIX "ll"
|
||||
# define FMTPTR_PREFIX "ll"
|
||||
# else
|
||||
# define FMT64_PREFIX "ll"
|
||||
# define FMTPTR_PREFIX ""
|
||||
# endif
|
||||
# define FMTd32 "d"
|
||||
# define FMTu32 "u"
|
||||
# define FMTx32 "x"
|
||||
# define FMTd64 FMT64_PREFIX "d"
|
||||
# define FMTu64 FMT64_PREFIX "u"
|
||||
# define FMTx64 FMT64_PREFIX "x"
|
||||
# define FMTdPTR FMTPTR_PREFIX "d"
|
||||
# define FMTuPTR FMTPTR_PREFIX "u"
|
||||
# define FMTxPTR FMTPTR_PREFIX "x"
|
||||
#else
|
||||
# include <inttypes.h>
|
||||
# define FMTd32 PRId32
|
||||
# define FMTu32 PRIu32
|
||||
# define FMTx32 PRIx32
|
||||
# define FMTd64 PRId64
|
||||
# define FMTu64 PRIu64
|
||||
# define FMTx64 PRIx64
|
||||
# define FMTdPTR PRIdPTR
|
||||
# define FMTuPTR PRIuPTR
|
||||
# define FMTxPTR PRIxPTR
|
||||
#endif
|
||||
|
||||
/* Size of stack-allocated buffer passed to buferror(). */
|
||||
#define BUFERROR_BUF 64
|
||||
|
||||
@@ -40,14 +10,6 @@
|
||||
*/
|
||||
#define MALLOC_PRINTF_BUFSIZE 4096
|
||||
|
||||
/* Junk fill patterns. */
|
||||
#ifndef JEMALLOC_ALLOC_JUNK
|
||||
# define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5)
|
||||
#endif
|
||||
#ifndef JEMALLOC_FREE_JUNK
|
||||
# define JEMALLOC_FREE_JUNK ((uint8_t)0x5a)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Wrap a cpp argument that contains commas such that it isn't broken up into
|
||||
* multiple arguments.
|
||||
@@ -60,30 +22,57 @@
|
||||
* uninitialized.
|
||||
*/
|
||||
#ifdef JEMALLOC_CC_SILENCE
|
||||
# define JEMALLOC_CC_SILENCE_INIT(v) = v
|
||||
# define JEMALLOC_CC_SILENCE_INIT(v) = v
|
||||
#else
|
||||
# define JEMALLOC_CC_SILENCE_INIT(v)
|
||||
# define JEMALLOC_CC_SILENCE_INIT(v)
|
||||
#endif
|
||||
|
||||
#ifdef __GNUC__
|
||||
# define likely(x) __builtin_expect(!!(x), 1)
|
||||
# define unlikely(x) __builtin_expect(!!(x), 0)
|
||||
#else
|
||||
# define likely(x) !!(x)
|
||||
# define unlikely(x) !!(x)
|
||||
/*
|
||||
* Define a custom assert() in order to reduce the chances of deadlock during
|
||||
* assertion failure.
|
||||
*/
|
||||
#ifndef assert
|
||||
#define assert(e) do { \
|
||||
if (config_debug && !(e)) { \
|
||||
malloc_printf( \
|
||||
"<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", \
|
||||
__FILE__, __LINE__, #e); \
|
||||
abort(); \
|
||||
} \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#if !defined(JEMALLOC_INTERNAL_UNREACHABLE)
|
||||
# error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure
|
||||
#ifndef not_reached
|
||||
#define not_reached() do { \
|
||||
if (config_debug) { \
|
||||
malloc_printf( \
|
||||
"<jemalloc>: %s:%d: Unreachable code reached\n", \
|
||||
__FILE__, __LINE__); \
|
||||
abort(); \
|
||||
} \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#define unreachable() JEMALLOC_INTERNAL_UNREACHABLE()
|
||||
#ifndef not_implemented
|
||||
#define not_implemented() do { \
|
||||
if (config_debug) { \
|
||||
malloc_printf("<jemalloc>: %s:%d: Not implemented\n", \
|
||||
__FILE__, __LINE__); \
|
||||
abort(); \
|
||||
} \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#include "jemalloc/internal/assert.h"
|
||||
#ifndef assert_not_implemented
|
||||
#define assert_not_implemented(e) do { \
|
||||
if (config_debug && !(e)) \
|
||||
not_implemented(); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
/* Use to assert a particular configuration, e.g., cassert(config_debug). */
|
||||
#define cassert(c) do { \
|
||||
if (unlikely(!(c))) \
|
||||
if ((c) == false) \
|
||||
not_reached(); \
|
||||
} while (0)
|
||||
|
||||
@@ -104,216 +93,47 @@ void malloc_write(const char *s);
|
||||
* malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating
|
||||
* point math.
|
||||
*/
|
||||
size_t malloc_vsnprintf(char *str, size_t size, const char *format,
|
||||
int malloc_vsnprintf(char *str, size_t size, const char *format,
|
||||
va_list ap);
|
||||
size_t malloc_snprintf(char *str, size_t size, const char *format, ...)
|
||||
JEMALLOC_FORMAT_PRINTF(3, 4);
|
||||
int malloc_snprintf(char *str, size_t size, const char *format, ...)
|
||||
JEMALLOC_ATTR(format(printf, 3, 4));
|
||||
void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
const char *format, va_list ap);
|
||||
void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque,
|
||||
const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4);
|
||||
void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
|
||||
const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4));
|
||||
void malloc_printf(const char *format, ...)
|
||||
JEMALLOC_ATTR(format(printf, 1, 2));
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
unsigned ffs_llu(unsigned long long bitmap);
|
||||
unsigned ffs_lu(unsigned long bitmap);
|
||||
unsigned ffs_u(unsigned bitmap);
|
||||
unsigned ffs_zu(size_t bitmap);
|
||||
unsigned ffs_u64(uint64_t bitmap);
|
||||
unsigned ffs_u32(uint32_t bitmap);
|
||||
uint64_t pow2_ceil_u64(uint64_t x);
|
||||
uint32_t pow2_ceil_u32(uint32_t x);
|
||||
size_t pow2_ceil_zu(size_t x);
|
||||
unsigned lg_floor(size_t x);
|
||||
size_t pow2_ceil(size_t x);
|
||||
void set_errno(int errnum);
|
||||
int get_errno(void);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_))
|
||||
|
||||
/* Sanity check. */
|
||||
#if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \
|
||||
|| !defined(JEMALLOC_INTERNAL_FFS)
|
||||
# error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure
|
||||
#endif
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE unsigned
|
||||
ffs_llu(unsigned long long bitmap)
|
||||
{
|
||||
|
||||
return (JEMALLOC_INTERNAL_FFSLL(bitmap));
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE unsigned
|
||||
ffs_lu(unsigned long bitmap)
|
||||
{
|
||||
|
||||
return (JEMALLOC_INTERNAL_FFSL(bitmap));
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE unsigned
|
||||
ffs_u(unsigned bitmap)
|
||||
{
|
||||
|
||||
return (JEMALLOC_INTERNAL_FFS(bitmap));
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE unsigned
|
||||
ffs_zu(size_t bitmap)
|
||||
{
|
||||
|
||||
#if LG_SIZEOF_PTR == LG_SIZEOF_INT
|
||||
return (ffs_u(bitmap));
|
||||
#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG
|
||||
return (ffs_lu(bitmap));
|
||||
#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG
|
||||
return (ffs_llu(bitmap));
|
||||
#else
|
||||
#error No implementation for size_t ffs()
|
||||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE unsigned
|
||||
ffs_u64(uint64_t bitmap)
|
||||
{
|
||||
|
||||
#if LG_SIZEOF_LONG == 3
|
||||
return (ffs_lu(bitmap));
|
||||
#elif LG_SIZEOF_LONG_LONG == 3
|
||||
return (ffs_llu(bitmap));
|
||||
#else
|
||||
#error No implementation for 64-bit ffs()
|
||||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE unsigned
|
||||
ffs_u32(uint32_t bitmap)
|
||||
{
|
||||
|
||||
#if LG_SIZEOF_INT == 2
|
||||
return (ffs_u(bitmap));
|
||||
#else
|
||||
#error No implementation for 32-bit ffs()
|
||||
#endif
|
||||
return (ffs_u(bitmap));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint64_t
|
||||
pow2_ceil_u64(uint64_t x)
|
||||
{
|
||||
|
||||
x--;
|
||||
x |= x >> 1;
|
||||
x |= x >> 2;
|
||||
x |= x >> 4;
|
||||
x |= x >> 8;
|
||||
x |= x >> 16;
|
||||
x |= x >> 32;
|
||||
x++;
|
||||
return (x);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE uint32_t
|
||||
pow2_ceil_u32(uint32_t x)
|
||||
{
|
||||
|
||||
x--;
|
||||
x |= x >> 1;
|
||||
x |= x >> 2;
|
||||
x |= x >> 4;
|
||||
x |= x >> 8;
|
||||
x |= x >> 16;
|
||||
x++;
|
||||
return (x);
|
||||
}
|
||||
|
||||
/* Compute the smallest power of 2 that is >= x. */
|
||||
JEMALLOC_INLINE size_t
|
||||
pow2_ceil_zu(size_t x)
|
||||
pow2_ceil(size_t x)
|
||||
{
|
||||
|
||||
x--;
|
||||
x |= x >> 1;
|
||||
x |= x >> 2;
|
||||
x |= x >> 4;
|
||||
x |= x >> 8;
|
||||
x |= x >> 16;
|
||||
#if (LG_SIZEOF_PTR == 3)
|
||||
return (pow2_ceil_u64(x));
|
||||
#else
|
||||
return (pow2_ceil_u32(x));
|
||||
x |= x >> 32;
|
||||
#endif
|
||||
}
|
||||
|
||||
#if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
|
||||
JEMALLOC_INLINE unsigned
|
||||
lg_floor(size_t x)
|
||||
{
|
||||
size_t ret;
|
||||
|
||||
assert(x != 0);
|
||||
|
||||
asm ("bsr %1, %0"
|
||||
: "=r"(ret) // Outputs.
|
||||
: "r"(x) // Inputs.
|
||||
);
|
||||
assert(ret < UINT_MAX);
|
||||
return ((unsigned)ret);
|
||||
}
|
||||
#elif (defined(_MSC_VER))
|
||||
JEMALLOC_INLINE unsigned
|
||||
lg_floor(size_t x)
|
||||
{
|
||||
unsigned long ret;
|
||||
|
||||
assert(x != 0);
|
||||
|
||||
#if (LG_SIZEOF_PTR == 3)
|
||||
_BitScanReverse64(&ret, x);
|
||||
#elif (LG_SIZEOF_PTR == 2)
|
||||
_BitScanReverse(&ret, x);
|
||||
#else
|
||||
# error "Unsupported type size for lg_floor()"
|
||||
#endif
|
||||
assert(ret < UINT_MAX);
|
||||
return ((unsigned)ret);
|
||||
}
|
||||
#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
|
||||
JEMALLOC_INLINE unsigned
|
||||
lg_floor(size_t x)
|
||||
{
|
||||
|
||||
assert(x != 0);
|
||||
|
||||
#if (LG_SIZEOF_PTR == LG_SIZEOF_INT)
|
||||
return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x));
|
||||
#elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG)
|
||||
return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x));
|
||||
#else
|
||||
# error "Unsupported type size for lg_floor()"
|
||||
#endif
|
||||
}
|
||||
#else
|
||||
JEMALLOC_INLINE unsigned
|
||||
lg_floor(size_t x)
|
||||
{
|
||||
|
||||
assert(x != 0);
|
||||
|
||||
x |= (x >> 1);
|
||||
x |= (x >> 2);
|
||||
x |= (x >> 4);
|
||||
x |= (x >> 8);
|
||||
x |= (x >> 16);
|
||||
#if (LG_SIZEOF_PTR == 3)
|
||||
x |= (x >> 32);
|
||||
#endif
|
||||
if (x == SIZE_T_MAX)
|
||||
return ((8 << LG_SIZEOF_PTR) - 1);
|
||||
x++;
|
||||
return (ffs_zu(x) - 2);
|
||||
return (x);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Set error code. */
|
||||
/* Sets error code */
|
||||
JEMALLOC_INLINE void
|
||||
set_errno(int errnum)
|
||||
{
|
||||
@@ -325,7 +145,7 @@ set_errno(int errnum)
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Get last error code. */
|
||||
/* Get last error code */
|
||||
JEMALLOC_INLINE int
|
||||
get_errno(void)
|
||||
{
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
#ifdef JEMALLOC_VALGRIND
|
||||
#include <valgrind/valgrind.h>
|
||||
|
||||
/*
|
||||
* The size that is reported to Valgrind must be consistent through a chain of
|
||||
* malloc..realloc..realloc calls. Request size isn't recorded anywhere in
|
||||
* jemalloc, so it is critical that all callers of these macros provide usize
|
||||
* rather than request size. As a result, buffer overflow detection is
|
||||
* technically weakened for the standard API, though it is generally accepted
|
||||
* practice to consider any extra bytes reported by malloc_usable_size() as
|
||||
* usable space.
|
||||
*/
|
||||
#define JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(ptr, usize) do { \
|
||||
if (unlikely(in_valgrind)) \
|
||||
valgrind_make_mem_noaccess(ptr, usize); \
|
||||
} while (0)
|
||||
#define JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ptr, usize) do { \
|
||||
if (unlikely(in_valgrind)) \
|
||||
valgrind_make_mem_undefined(ptr, usize); \
|
||||
} while (0)
|
||||
#define JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ptr, usize) do { \
|
||||
if (unlikely(in_valgrind)) \
|
||||
valgrind_make_mem_defined(ptr, usize); \
|
||||
} while (0)
|
||||
/*
|
||||
* The VALGRIND_MALLOCLIKE_BLOCK() and VALGRIND_RESIZEINPLACE_BLOCK() macro
|
||||
* calls must be embedded in macros rather than in functions so that when
|
||||
* Valgrind reports errors, there are no extra stack frames in the backtraces.
|
||||
*/
|
||||
#define JEMALLOC_VALGRIND_MALLOC(cond, tsdn, ptr, usize, zero) do { \
|
||||
if (unlikely(in_valgrind && cond)) { \
|
||||
VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(tsdn, ptr), \
|
||||
zero); \
|
||||
} \
|
||||
} while (0)
|
||||
#define JEMALLOC_VALGRIND_REALLOC_MOVED_no(ptr, old_ptr) \
|
||||
(false)
|
||||
#define JEMALLOC_VALGRIND_REALLOC_MOVED_maybe(ptr, old_ptr) \
|
||||
((ptr) != (old_ptr))
|
||||
#define JEMALLOC_VALGRIND_REALLOC_PTR_NULL_no(ptr) \
|
||||
(false)
|
||||
#define JEMALLOC_VALGRIND_REALLOC_PTR_NULL_maybe(ptr) \
|
||||
(ptr == NULL)
|
||||
#define JEMALLOC_VALGRIND_REALLOC_OLD_PTR_NULL_no(old_ptr) \
|
||||
(false)
|
||||
#define JEMALLOC_VALGRIND_REALLOC_OLD_PTR_NULL_maybe(old_ptr) \
|
||||
(old_ptr == NULL)
|
||||
#define JEMALLOC_VALGRIND_REALLOC(moved, tsdn, ptr, usize, ptr_null, \
|
||||
old_ptr, old_usize, old_rzsize, old_ptr_null, zero) do { \
|
||||
if (unlikely(in_valgrind)) { \
|
||||
size_t rzsize = p2rz(tsdn, ptr); \
|
||||
\
|
||||
if (!JEMALLOC_VALGRIND_REALLOC_MOVED_##moved(ptr, \
|
||||
old_ptr)) { \
|
||||
VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \
|
||||
usize, rzsize); \
|
||||
if (zero && old_usize < usize) { \
|
||||
valgrind_make_mem_defined( \
|
||||
(void *)((uintptr_t)ptr + \
|
||||
old_usize), usize - old_usize); \
|
||||
} \
|
||||
} else { \
|
||||
if (!JEMALLOC_VALGRIND_REALLOC_OLD_PTR_NULL_## \
|
||||
old_ptr_null(old_ptr)) { \
|
||||
valgrind_freelike_block(old_ptr, \
|
||||
old_rzsize); \
|
||||
} \
|
||||
if (!JEMALLOC_VALGRIND_REALLOC_PTR_NULL_## \
|
||||
ptr_null(ptr)) { \
|
||||
size_t copy_size = (old_usize < usize) \
|
||||
? old_usize : usize; \
|
||||
size_t tail_size = usize - copy_size; \
|
||||
VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, \
|
||||
rzsize, false); \
|
||||
if (copy_size > 0) { \
|
||||
valgrind_make_mem_defined(ptr, \
|
||||
copy_size); \
|
||||
} \
|
||||
if (zero && tail_size > 0) { \
|
||||
valgrind_make_mem_defined( \
|
||||
(void *)((uintptr_t)ptr + \
|
||||
copy_size), tail_size); \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \
|
||||
if (unlikely(in_valgrind)) \
|
||||
valgrind_freelike_block(ptr, rzsize); \
|
||||
} while (0)
|
||||
#else
|
||||
#define RUNNING_ON_VALGRIND ((unsigned)0)
|
||||
#define JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(ptr, usize) do {} while (0)
|
||||
#define JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ptr, usize) do {} while (0)
|
||||
#define JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ptr, usize) do {} while (0)
|
||||
#define JEMALLOC_VALGRIND_MALLOC(cond, tsdn, ptr, usize, zero) do {} while (0)
|
||||
#define JEMALLOC_VALGRIND_REALLOC(maybe_moved, tsdn, ptr, usize, \
|
||||
ptr_maybe_null, old_ptr, old_usize, old_rzsize, old_ptr_maybe_null, \
|
||||
zero) do {} while (0)
|
||||
#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do {} while (0)
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
#ifdef JEMALLOC_VALGRIND
|
||||
void valgrind_make_mem_noaccess(void *ptr, size_t usize);
|
||||
void valgrind_make_mem_undefined(void *ptr, size_t usize);
|
||||
void valgrind_make_mem_defined(void *ptr, size_t usize);
|
||||
void valgrind_freelike_block(void *ptr, size_t usize);
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
|
||||
@@ -1,266 +0,0 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
typedef struct witness_s witness_t;
|
||||
typedef unsigned witness_rank_t;
|
||||
typedef ql_head(witness_t) witness_list_t;
|
||||
typedef int witness_comp_t (const witness_t *, const witness_t *);
|
||||
|
||||
/*
|
||||
* Lock ranks. Witnesses with rank WITNESS_RANK_OMIT are completely ignored by
|
||||
* the witness machinery.
|
||||
*/
|
||||
#define WITNESS_RANK_OMIT 0U
|
||||
|
||||
#define WITNESS_RANK_INIT 1U
|
||||
#define WITNESS_RANK_CTL 1U
|
||||
#define WITNESS_RANK_ARENAS 2U
|
||||
|
||||
#define WITNESS_RANK_PROF_DUMP 3U
|
||||
#define WITNESS_RANK_PROF_BT2GCTX 4U
|
||||
#define WITNESS_RANK_PROF_TDATAS 5U
|
||||
#define WITNESS_RANK_PROF_TDATA 6U
|
||||
#define WITNESS_RANK_PROF_GCTX 7U
|
||||
|
||||
#define WITNESS_RANK_ARENA 8U
|
||||
#define WITNESS_RANK_ARENA_CHUNKS 9U
|
||||
#define WITNESS_RANK_ARENA_NODE_CACHE 10
|
||||
|
||||
#define WITNESS_RANK_BASE 11U
|
||||
|
||||
#define WITNESS_RANK_LEAF 0xffffffffU
|
||||
#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF
|
||||
#define WITNESS_RANK_ARENA_HUGE WITNESS_RANK_LEAF
|
||||
#define WITNESS_RANK_DSS WITNESS_RANK_LEAF
|
||||
#define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF
|
||||
#define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF
|
||||
#define WITNESS_RANK_PROF_GDUMP WITNESS_RANK_LEAF
|
||||
#define WITNESS_RANK_PROF_NEXT_THR_UID WITNESS_RANK_LEAF
|
||||
#define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT WITNESS_RANK_LEAF
|
||||
|
||||
#define WITNESS_INITIALIZER(rank) {"initializer", rank, NULL, {NULL, NULL}}
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
|
||||
struct witness_s {
|
||||
/* Name, used for printing lock order reversal messages. */
|
||||
const char *name;
|
||||
|
||||
/*
|
||||
* Witness rank, where 0 is lowest and UINT_MAX is highest. Witnesses
|
||||
* must be acquired in order of increasing rank.
|
||||
*/
|
||||
witness_rank_t rank;
|
||||
|
||||
/*
|
||||
* If two witnesses are of equal rank and they have the samp comp
|
||||
* function pointer, it is called as a last attempt to differentiate
|
||||
* between witnesses of equal rank.
|
||||
*/
|
||||
witness_comp_t *comp;
|
||||
|
||||
/* Linkage for thread's currently owned locks. */
|
||||
ql_elm(witness_t) link;
|
||||
};
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
void witness_init(witness_t *witness, const char *name, witness_rank_t rank,
|
||||
witness_comp_t *comp);
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *);
|
||||
extern witness_lock_error_t *witness_lock_error;
|
||||
#else
|
||||
void witness_lock_error(const witness_list_t *witnesses,
|
||||
const witness_t *witness);
|
||||
#endif
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef void (witness_owner_error_t)(const witness_t *);
|
||||
extern witness_owner_error_t *witness_owner_error;
|
||||
#else
|
||||
void witness_owner_error(const witness_t *witness);
|
||||
#endif
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef void (witness_not_owner_error_t)(const witness_t *);
|
||||
extern witness_not_owner_error_t *witness_not_owner_error;
|
||||
#else
|
||||
void witness_not_owner_error(const witness_t *witness);
|
||||
#endif
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef void (witness_lockless_error_t)(const witness_list_t *);
|
||||
extern witness_lockless_error_t *witness_lockless_error;
|
||||
#else
|
||||
void witness_lockless_error(const witness_list_t *witnesses);
|
||||
#endif
|
||||
|
||||
void witnesses_cleanup(tsd_t *tsd);
|
||||
void witness_fork_cleanup(tsd_t *tsd);
|
||||
void witness_prefork(tsd_t *tsd);
|
||||
void witness_postfork_parent(tsd_t *tsd);
|
||||
void witness_postfork_child(tsd_t *tsd);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
bool witness_owner(tsd_t *tsd, const witness_t *witness);
|
||||
void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
|
||||
void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
|
||||
void witness_assert_lockless(tsdn_t *tsdn);
|
||||
void witness_lock(tsdn_t *tsdn, witness_t *witness);
|
||||
void witness_unlock(tsdn_t *tsdn, witness_t *witness);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
|
||||
JEMALLOC_INLINE bool
|
||||
witness_owner(tsd_t *tsd, const witness_t *witness)
|
||||
{
|
||||
witness_list_t *witnesses;
|
||||
witness_t *w;
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
ql_foreach(w, witnesses, link) {
|
||||
if (w == witness)
|
||||
return (true);
|
||||
}
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
witness_assert_owner(tsdn_t *tsdn, const witness_t *witness)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
|
||||
if (!config_debug)
|
||||
return;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
if (witness->rank == WITNESS_RANK_OMIT)
|
||||
return;
|
||||
|
||||
if (witness_owner(tsd, witness))
|
||||
return;
|
||||
witness_owner_error(witness);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
witness_t *w;
|
||||
|
||||
if (!config_debug)
|
||||
return;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
if (witness->rank == WITNESS_RANK_OMIT)
|
||||
return;
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
ql_foreach(w, witnesses, link) {
|
||||
if (w == witness)
|
||||
witness_not_owner_error(witness);
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
witness_assert_lockless(tsdn_t *tsdn)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
witness_t *w;
|
||||
|
||||
if (!config_debug)
|
||||
return;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
w = ql_last(witnesses, link);
|
||||
if (w != NULL)
|
||||
witness_lockless_error(witnesses);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
witness_lock(tsdn_t *tsdn, witness_t *witness)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
witness_t *w;
|
||||
|
||||
if (!config_debug)
|
||||
return;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
if (witness->rank == WITNESS_RANK_OMIT)
|
||||
return;
|
||||
|
||||
witness_assert_not_owner(tsdn, witness);
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
w = ql_last(witnesses, link);
|
||||
if (w == NULL) {
|
||||
/* No other locks; do nothing. */
|
||||
} else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) {
|
||||
/* Forking, and relaxed ranking satisfied. */
|
||||
} else if (w->rank > witness->rank) {
|
||||
/* Not forking, rank order reversal. */
|
||||
witness_lock_error(witnesses, witness);
|
||||
} else if (w->rank == witness->rank && (w->comp == NULL || w->comp !=
|
||||
witness->comp || w->comp(w, witness) > 0)) {
|
||||
/*
|
||||
* Missing/incompatible comparison function, or comparison
|
||||
* function indicates rank order reversal.
|
||||
*/
|
||||
witness_lock_error(witnesses, witness);
|
||||
}
|
||||
|
||||
ql_elm_new(witness, link);
|
||||
ql_tail_insert(witnesses, witness, link);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
witness_unlock(tsdn_t *tsdn, witness_t *witness)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
|
||||
if (!config_debug)
|
||||
return;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
if (witness->rank == WITNESS_RANK_OMIT)
|
||||
return;
|
||||
|
||||
/*
|
||||
* Check whether owner before removal, rather than relying on
|
||||
* witness_assert_owner() to abort, so that unit tests can test this
|
||||
* function's failure mode without causing undefined behavior.
|
||||
*/
|
||||
if (witness_owner(tsd, witness)) {
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
ql_remove(witnesses, witness, link);
|
||||
} else
|
||||
witness_assert_owner(tsdn, witness);
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
@@ -12,7 +12,7 @@ extern "C" {
|
||||
EOF
|
||||
|
||||
for hdr in jemalloc_defs.h jemalloc_rename.h jemalloc_macros.h \
|
||||
jemalloc_protos.h jemalloc_typedefs.h jemalloc_mangle.h ; do
|
||||
jemalloc_protos.h jemalloc_mangle.h ; do
|
||||
cat "${objroot}include/jemalloc/${hdr}" \
|
||||
| grep -v 'Generated from .* by configure\.' \
|
||||
| sed -e 's/^#define /#define /g' \
|
||||
@@ -22,7 +22,7 @@ done
|
||||
|
||||
cat <<EOF
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
};
|
||||
#endif
|
||||
#endif /* JEMALLOC_H_ */
|
||||
EOF
|
||||
|
||||
@@ -1,14 +1,8 @@
|
||||
/* Defined if __attribute__((...)) syntax is supported. */
|
||||
#undef JEMALLOC_HAVE_ATTR
|
||||
|
||||
/* Defined if alloc_size attribute is supported. */
|
||||
#undef JEMALLOC_HAVE_ATTR_ALLOC_SIZE
|
||||
|
||||
/* Defined if format(gnu_printf, ...) attribute is supported. */
|
||||
#undef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF
|
||||
|
||||
/* Defined if format(printf, ...) attribute is supported. */
|
||||
#undef JEMALLOC_HAVE_ATTR_FORMAT_PRINTF
|
||||
/* Support the experimental API. */
|
||||
#undef JEMALLOC_EXPERIMENTAL
|
||||
|
||||
/*
|
||||
* Define overrides for non-standard allocator-related functions if they are
|
||||
@@ -26,20 +20,5 @@
|
||||
*/
|
||||
#undef JEMALLOC_USABLE_SIZE_CONST
|
||||
|
||||
/*
|
||||
* If defined, specify throw() for the public function prototypes when compiling
|
||||
* with C++. The only justification for this is to match the prototypes that
|
||||
* glibc defines.
|
||||
*/
|
||||
#undef JEMALLOC_USE_CXX_THROW
|
||||
|
||||
#ifdef _MSC_VER
|
||||
# ifdef _WIN64
|
||||
# define LG_SIZEOF_PTR_WIN 3
|
||||
# else
|
||||
# define LG_SIZEOF_PTR_WIN 2
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* sizeof(void *) == 2^LG_SIZEOF_PTR. */
|
||||
#undef LG_SIZEOF_PTR
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
#include <stdlib.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <limits.h>
|
||||
#include <strings.h>
|
||||
|
||||
@@ -11,93 +8,54 @@
|
||||
#define JEMALLOC_VERSION_NREV @jemalloc_version_nrev@
|
||||
#define JEMALLOC_VERSION_GID "@jemalloc_version_gid@"
|
||||
|
||||
# define MALLOCX_LG_ALIGN(la) ((int)(la))
|
||||
# define MALLOCX_LG_ALIGN(la) (la)
|
||||
# if LG_SIZEOF_PTR == 2
|
||||
# define MALLOCX_ALIGN(a) ((int)(ffs((int)(a))-1))
|
||||
# define MALLOCX_ALIGN(a) (ffs(a)-1)
|
||||
# else
|
||||
# define MALLOCX_ALIGN(a) \
|
||||
((int)(((size_t)(a) < (size_t)INT_MAX) ? ffs((int)(a))-1 : \
|
||||
ffs((int)(((size_t)(a))>>32))+31))
|
||||
((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31)
|
||||
# endif
|
||||
# define MALLOCX_ZERO ((int)0x40)
|
||||
/*
|
||||
* Bias tcache index bits so that 0 encodes "automatic tcache management", and 1
|
||||
* encodes MALLOCX_TCACHE_NONE.
|
||||
*/
|
||||
# define MALLOCX_TCACHE(tc) ((int)(((tc)+2) << 8))
|
||||
# define MALLOCX_TCACHE_NONE MALLOCX_TCACHE(-1)
|
||||
/*
|
||||
* Bias arena index bits so that 0 encodes "use an automatically chosen arena".
|
||||
*/
|
||||
# define MALLOCX_ARENA(a) ((((int)(a))+1) << 20)
|
||||
/* Bias arena index bits so that 0 encodes "MALLOCX_ARENA() unspecified". */
|
||||
# define MALLOCX_ARENA(a) ((int)(((a)+1) << 8))
|
||||
|
||||
#if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW)
|
||||
# define JEMALLOC_CXX_THROW throw()
|
||||
#else
|
||||
# define JEMALLOC_CXX_THROW
|
||||
#ifdef JEMALLOC_EXPERIMENTAL
|
||||
# define ALLOCM_LG_ALIGN(la) (la)
|
||||
# if LG_SIZEOF_PTR == 2
|
||||
# define ALLOCM_ALIGN(a) (ffs(a)-1)
|
||||
# else
|
||||
# define ALLOCM_ALIGN(a) \
|
||||
((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31)
|
||||
# endif
|
||||
# define ALLOCM_ZERO ((int)0x40)
|
||||
# define ALLOCM_NO_MOVE ((int)0x80)
|
||||
/* Bias arena index bits so that 0 encodes "ALLOCM_ARENA() unspecified". */
|
||||
# define ALLOCM_ARENA(a) ((int)(((a)+1) << 8))
|
||||
# define ALLOCM_SUCCESS 0
|
||||
# define ALLOCM_ERR_OOM 1
|
||||
# define ALLOCM_ERR_NOT_MOVED 2
|
||||
#endif
|
||||
|
||||
#if _MSC_VER
|
||||
# define JEMALLOC_ATTR(s)
|
||||
# define JEMALLOC_ALIGNED(s) __declspec(align(s))
|
||||
# define JEMALLOC_ALLOC_SIZE(s)
|
||||
# define JEMALLOC_ALLOC_SIZE2(s1, s2)
|
||||
# ifndef JEMALLOC_EXPORT
|
||||
# ifdef DLLEXPORT
|
||||
# define JEMALLOC_EXPORT __declspec(dllexport)
|
||||
# else
|
||||
# define JEMALLOC_EXPORT __declspec(dllimport)
|
||||
# endif
|
||||
# endif
|
||||
# define JEMALLOC_FORMAT_PRINTF(s, i)
|
||||
# define JEMALLOC_NOINLINE __declspec(noinline)
|
||||
# ifdef __cplusplus
|
||||
# define JEMALLOC_NOTHROW __declspec(nothrow)
|
||||
# else
|
||||
# define JEMALLOC_NOTHROW
|
||||
# endif
|
||||
# define JEMALLOC_SECTION(s) __declspec(allocate(s))
|
||||
# define JEMALLOC_RESTRICT_RETURN __declspec(restrict)
|
||||
# if _MSC_VER >= 1900 && !defined(__EDG__)
|
||||
# define JEMALLOC_ALLOCATOR __declspec(allocator)
|
||||
# else
|
||||
# define JEMALLOC_ALLOCATOR
|
||||
# endif
|
||||
#elif defined(JEMALLOC_HAVE_ATTR)
|
||||
#ifdef JEMALLOC_HAVE_ATTR
|
||||
# define JEMALLOC_ATTR(s) __attribute__((s))
|
||||
# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default"))
|
||||
# define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s))
|
||||
# ifdef JEMALLOC_HAVE_ATTR_ALLOC_SIZE
|
||||
# define JEMALLOC_ALLOC_SIZE(s) JEMALLOC_ATTR(alloc_size(s))
|
||||
# define JEMALLOC_ALLOC_SIZE2(s1, s2) JEMALLOC_ATTR(alloc_size(s1, s2))
|
||||
# else
|
||||
# define JEMALLOC_ALLOC_SIZE(s)
|
||||
# define JEMALLOC_ALLOC_SIZE2(s1, s2)
|
||||
# endif
|
||||
# ifndef JEMALLOC_EXPORT
|
||||
# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default"))
|
||||
# endif
|
||||
# ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF
|
||||
# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i))
|
||||
# elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF)
|
||||
# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(printf, s, i))
|
||||
# else
|
||||
# define JEMALLOC_FORMAT_PRINTF(s, i)
|
||||
# endif
|
||||
# define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline)
|
||||
# define JEMALLOC_NOTHROW JEMALLOC_ATTR(nothrow)
|
||||
# define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s))
|
||||
# define JEMALLOC_RESTRICT_RETURN
|
||||
# define JEMALLOC_ALLOCATOR
|
||||
# define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline)
|
||||
#elif _MSC_VER
|
||||
# define JEMALLOC_ATTR(s)
|
||||
# ifdef DLLEXPORT
|
||||
# define JEMALLOC_EXPORT __declspec(dllexport)
|
||||
# else
|
||||
# define JEMALLOC_EXPORT __declspec(dllimport)
|
||||
# endif
|
||||
# define JEMALLOC_ALIGNED(s) __declspec(align(s))
|
||||
# define JEMALLOC_SECTION(s) __declspec(allocate(s))
|
||||
# define JEMALLOC_NOINLINE __declspec(noinline)
|
||||
#else
|
||||
# define JEMALLOC_ATTR(s)
|
||||
# define JEMALLOC_ALIGNED(s)
|
||||
# define JEMALLOC_ALLOC_SIZE(s)
|
||||
# define JEMALLOC_ALLOC_SIZE2(s1, s2)
|
||||
# define JEMALLOC_EXPORT
|
||||
# define JEMALLOC_FORMAT_PRINTF(s, i)
|
||||
# define JEMALLOC_NOINLINE
|
||||
# define JEMALLOC_NOTHROW
|
||||
# define JEMALLOC_ALIGNED(s)
|
||||
# define JEMALLOC_SECTION(s)
|
||||
# define JEMALLOC_RESTRICT_RETURN
|
||||
# define JEMALLOC_ALLOCATOR
|
||||
# define JEMALLOC_NOINLINE
|
||||
#endif
|
||||
|
||||
@@ -7,60 +7,52 @@ extern JEMALLOC_EXPORT const char *@je_@malloc_conf;
|
||||
extern JEMALLOC_EXPORT void (*@je_@malloc_message)(void *cbopaque,
|
||||
const char *s);
|
||||
|
||||
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
|
||||
void JEMALLOC_NOTHROW *@je_@malloc(size_t size)
|
||||
JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1);
|
||||
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
|
||||
void JEMALLOC_NOTHROW *@je_@calloc(size_t num, size_t size)
|
||||
JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2);
|
||||
JEMALLOC_EXPORT int JEMALLOC_NOTHROW @je_@posix_memalign(void **memptr,
|
||||
size_t alignment, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(nonnull(1));
|
||||
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
|
||||
void JEMALLOC_NOTHROW *@je_@aligned_alloc(size_t alignment,
|
||||
size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc)
|
||||
JEMALLOC_ALLOC_SIZE(2);
|
||||
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
|
||||
void JEMALLOC_NOTHROW *@je_@realloc(void *ptr, size_t size)
|
||||
JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2);
|
||||
JEMALLOC_EXPORT void JEMALLOC_NOTHROW @je_@free(void *ptr)
|
||||
JEMALLOC_CXX_THROW;
|
||||
JEMALLOC_EXPORT void *@je_@malloc(size_t size) JEMALLOC_ATTR(malloc);
|
||||
JEMALLOC_EXPORT void *@je_@calloc(size_t num, size_t size)
|
||||
JEMALLOC_ATTR(malloc);
|
||||
JEMALLOC_EXPORT int @je_@posix_memalign(void **memptr, size_t alignment,
|
||||
size_t size) JEMALLOC_ATTR(nonnull(1));
|
||||
JEMALLOC_EXPORT void *@je_@aligned_alloc(size_t alignment, size_t size)
|
||||
JEMALLOC_ATTR(malloc);
|
||||
JEMALLOC_EXPORT void *@je_@realloc(void *ptr, size_t size);
|
||||
JEMALLOC_EXPORT void @je_@free(void *ptr);
|
||||
|
||||
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
|
||||
void JEMALLOC_NOTHROW *@je_@mallocx(size_t size, int flags)
|
||||
JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1);
|
||||
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
|
||||
void JEMALLOC_NOTHROW *@je_@rallocx(void *ptr, size_t size,
|
||||
int flags) JEMALLOC_ALLOC_SIZE(2);
|
||||
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @je_@xallocx(void *ptr, size_t size,
|
||||
size_t extra, int flags);
|
||||
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @je_@sallocx(const void *ptr,
|
||||
int flags) JEMALLOC_ATTR(pure);
|
||||
JEMALLOC_EXPORT void JEMALLOC_NOTHROW @je_@dallocx(void *ptr, int flags);
|
||||
JEMALLOC_EXPORT void JEMALLOC_NOTHROW @je_@sdallocx(void *ptr, size_t size,
|
||||
JEMALLOC_EXPORT void *@je_@mallocx(size_t size, int flags);
|
||||
JEMALLOC_EXPORT void *@je_@rallocx(void *ptr, size_t size, int flags);
|
||||
JEMALLOC_EXPORT size_t @je_@xallocx(void *ptr, size_t size, size_t extra,
|
||||
int flags);
|
||||
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @je_@nallocx(size_t size, int flags)
|
||||
JEMALLOC_ATTR(pure);
|
||||
JEMALLOC_EXPORT size_t @je_@sallocx(const void *ptr, int flags);
|
||||
JEMALLOC_EXPORT void @je_@dallocx(void *ptr, int flags);
|
||||
JEMALLOC_EXPORT size_t @je_@nallocx(size_t size, int flags);
|
||||
|
||||
JEMALLOC_EXPORT int JEMALLOC_NOTHROW @je_@mallctl(const char *name,
|
||||
JEMALLOC_EXPORT int @je_@mallctl(const char *name, void *oldp,
|
||||
size_t *oldlenp, void *newp, size_t newlen);
|
||||
JEMALLOC_EXPORT int @je_@mallctlnametomib(const char *name, size_t *mibp,
|
||||
size_t *miblenp);
|
||||
JEMALLOC_EXPORT int @je_@mallctlbymib(const size_t *mib, size_t miblen,
|
||||
void *oldp, size_t *oldlenp, void *newp, size_t newlen);
|
||||
JEMALLOC_EXPORT int JEMALLOC_NOTHROW @je_@mallctlnametomib(const char *name,
|
||||
size_t *mibp, size_t *miblenp);
|
||||
JEMALLOC_EXPORT int JEMALLOC_NOTHROW @je_@mallctlbymib(const size_t *mib,
|
||||
size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
|
||||
JEMALLOC_EXPORT void JEMALLOC_NOTHROW @je_@malloc_stats_print(
|
||||
void (*write_cb)(void *, const char *), void *@je_@cbopaque,
|
||||
const char *opts);
|
||||
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @je_@malloc_usable_size(
|
||||
JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW;
|
||||
JEMALLOC_EXPORT void @je_@malloc_stats_print(void (*write_cb)(void *,
|
||||
const char *), void *@je_@cbopaque, const char *opts);
|
||||
JEMALLOC_EXPORT size_t @je_@malloc_usable_size(
|
||||
JEMALLOC_USABLE_SIZE_CONST void *ptr);
|
||||
|
||||
#ifdef JEMALLOC_OVERRIDE_MEMALIGN
|
||||
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
|
||||
void JEMALLOC_NOTHROW *@je_@memalign(size_t alignment, size_t size)
|
||||
JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc);
|
||||
JEMALLOC_EXPORT void * @je_@memalign(size_t alignment, size_t size)
|
||||
JEMALLOC_ATTR(malloc);
|
||||
#endif
|
||||
|
||||
#ifdef JEMALLOC_OVERRIDE_VALLOC
|
||||
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
|
||||
void JEMALLOC_NOTHROW *@je_@valloc(size_t size) JEMALLOC_CXX_THROW
|
||||
JEMALLOC_ATTR(malloc);
|
||||
JEMALLOC_EXPORT void * @je_@valloc(size_t size) JEMALLOC_ATTR(malloc);
|
||||
#endif
|
||||
|
||||
#ifdef JEMALLOC_EXPERIMENTAL
|
||||
JEMALLOC_EXPORT int @je_@allocm(void **ptr, size_t *rsize, size_t size,
|
||||
int flags) JEMALLOC_ATTR(nonnull(1));
|
||||
JEMALLOC_EXPORT int @je_@rallocm(void **ptr, size_t *rsize, size_t size,
|
||||
size_t extra, int flags) JEMALLOC_ATTR(nonnull(1));
|
||||
JEMALLOC_EXPORT int @je_@sallocm(const void *ptr, size_t *rsize, int flags)
|
||||
JEMALLOC_ATTR(nonnull(1));
|
||||
JEMALLOC_EXPORT int @je_@dallocm(void *ptr, int flags)
|
||||
JEMALLOC_ATTR(nonnull(1));
|
||||
JEMALLOC_EXPORT int @je_@nallocm(size_t *rsize, size_t size, int flags);
|
||||
#endif
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
/*
|
||||
* void *
|
||||
* chunk_alloc(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
* bool *commit, unsigned arena_ind);
|
||||
*/
|
||||
typedef void *(chunk_alloc_t)(void *, size_t, size_t, bool *, bool *, unsigned);
|
||||
|
||||
/*
|
||||
* bool
|
||||
* chunk_dalloc(void *chunk, size_t size, bool committed, unsigned arena_ind);
|
||||
*/
|
||||
typedef bool (chunk_dalloc_t)(void *, size_t, bool, unsigned);
|
||||
|
||||
/*
|
||||
* bool
|
||||
* chunk_commit(void *chunk, size_t size, size_t offset, size_t length,
|
||||
* unsigned arena_ind);
|
||||
*/
|
||||
typedef bool (chunk_commit_t)(void *, size_t, size_t, size_t, unsigned);
|
||||
|
||||
/*
|
||||
* bool
|
||||
* chunk_decommit(void *chunk, size_t size, size_t offset, size_t length,
|
||||
* unsigned arena_ind);
|
||||
*/
|
||||
typedef bool (chunk_decommit_t)(void *, size_t, size_t, size_t, unsigned);
|
||||
|
||||
/*
|
||||
* bool
|
||||
* chunk_purge(void *chunk, size_t size, size_t offset, size_t length,
|
||||
* unsigned arena_ind);
|
||||
*/
|
||||
typedef bool (chunk_purge_t)(void *, size_t, size_t, size_t, unsigned);
|
||||
|
||||
/*
|
||||
* bool
|
||||
* chunk_split(void *chunk, size_t size, size_t size_a, size_t size_b,
|
||||
* bool committed, unsigned arena_ind);
|
||||
*/
|
||||
typedef bool (chunk_split_t)(void *, size_t, size_t, size_t, bool, unsigned);
|
||||
|
||||
/*
|
||||
* bool
|
||||
* chunk_merge(void *chunk_a, size_t size_a, void *chunk_b, size_t size_b,
|
||||
* bool committed, unsigned arena_ind);
|
||||
*/
|
||||
typedef bool (chunk_merge_t)(void *, size_t, void *, size_t, bool, unsigned);
|
||||
|
||||
typedef struct {
|
||||
chunk_alloc_t *alloc;
|
||||
chunk_dalloc_t *dalloc;
|
||||
chunk_commit_t *commit;
|
||||
chunk_decommit_t *decommit;
|
||||
chunk_purge_t *purge;
|
||||
chunk_split_t *split;
|
||||
chunk_merge_t *merge;
|
||||
} chunk_hooks_t;
|
||||
313
include/msvc_compat/inttypes.h
Normal file
313
include/msvc_compat/inttypes.h
Normal file
@@ -0,0 +1,313 @@
|
||||
// ISO C9x compliant inttypes.h for Microsoft Visual Studio
|
||||
// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124
|
||||
//
|
||||
// Copyright (c) 2006 Alexander Chemeris
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// 2. Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimer in the
|
||||
// documentation and/or other materials provided with the distribution.
|
||||
//
|
||||
// 3. The name of the author may be used to endorse or promote products
|
||||
// derived from this software without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
|
||||
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
||||
// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
|
||||
// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef _MSC_VER // [
|
||||
#error "Use this header only with Microsoft Visual C++ compilers!"
|
||||
#endif // _MSC_VER ]
|
||||
|
||||
#ifndef _MSC_INTTYPES_H_ // [
|
||||
#define _MSC_INTTYPES_H_
|
||||
|
||||
#if _MSC_VER > 1000
|
||||
#pragma once
|
||||
#endif
|
||||
|
||||
#include "stdint.h"
|
||||
|
||||
// 7.8 Format conversion of integer types
|
||||
|
||||
typedef struct {
|
||||
intmax_t quot;
|
||||
intmax_t rem;
|
||||
} imaxdiv_t;
|
||||
|
||||
// 7.8.1 Macros for format specifiers
|
||||
|
||||
#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198
|
||||
|
||||
#ifdef _WIN64
|
||||
# define __PRI64_PREFIX "l"
|
||||
# define __PRIPTR_PREFIX "l"
|
||||
#else
|
||||
# define __PRI64_PREFIX "ll"
|
||||
# define __PRIPTR_PREFIX
|
||||
#endif
|
||||
|
||||
// The fprintf macros for signed integers are:
|
||||
#define PRId8 "d"
|
||||
#define PRIi8 "i"
|
||||
#define PRIdLEAST8 "d"
|
||||
#define PRIiLEAST8 "i"
|
||||
#define PRIdFAST8 "d"
|
||||
#define PRIiFAST8 "i"
|
||||
|
||||
#define PRId16 "hd"
|
||||
#define PRIi16 "hi"
|
||||
#define PRIdLEAST16 "hd"
|
||||
#define PRIiLEAST16 "hi"
|
||||
#define PRIdFAST16 "hd"
|
||||
#define PRIiFAST16 "hi"
|
||||
|
||||
#define PRId32 "d"
|
||||
#define PRIi32 "i"
|
||||
#define PRIdLEAST32 "d"
|
||||
#define PRIiLEAST32 "i"
|
||||
#define PRIdFAST32 "d"
|
||||
#define PRIiFAST32 "i"
|
||||
|
||||
#define PRId64 __PRI64_PREFIX "d"
|
||||
#define PRIi64 __PRI64_PREFIX "i"
|
||||
#define PRIdLEAST64 __PRI64_PREFIX "d"
|
||||
#define PRIiLEAST64 __PRI64_PREFIX "i"
|
||||
#define PRIdFAST64 __PRI64_PREFIX "d"
|
||||
#define PRIiFAST64 __PRI64_PREFIX "i"
|
||||
|
||||
#define PRIdMAX __PRI64_PREFIX "d"
|
||||
#define PRIiMAX __PRI64_PREFIX "i"
|
||||
|
||||
#define PRIdPTR __PRIPTR_PREFIX "d"
|
||||
#define PRIiPTR __PRIPTR_PREFIX "i"
|
||||
|
||||
// The fprintf macros for unsigned integers are:
|
||||
#define PRIo8 "o"
|
||||
#define PRIu8 "u"
|
||||
#define PRIx8 "x"
|
||||
#define PRIX8 "X"
|
||||
#define PRIoLEAST8 "o"
|
||||
#define PRIuLEAST8 "u"
|
||||
#define PRIxLEAST8 "x"
|
||||
#define PRIXLEAST8 "X"
|
||||
#define PRIoFAST8 "o"
|
||||
#define PRIuFAST8 "u"
|
||||
#define PRIxFAST8 "x"
|
||||
#define PRIXFAST8 "X"
|
||||
|
||||
#define PRIo16 "ho"
|
||||
#define PRIu16 "hu"
|
||||
#define PRIx16 "hx"
|
||||
#define PRIX16 "hX"
|
||||
#define PRIoLEAST16 "ho"
|
||||
#define PRIuLEAST16 "hu"
|
||||
#define PRIxLEAST16 "hx"
|
||||
#define PRIXLEAST16 "hX"
|
||||
#define PRIoFAST16 "ho"
|
||||
#define PRIuFAST16 "hu"
|
||||
#define PRIxFAST16 "hx"
|
||||
#define PRIXFAST16 "hX"
|
||||
|
||||
#define PRIo32 "o"
|
||||
#define PRIu32 "u"
|
||||
#define PRIx32 "x"
|
||||
#define PRIX32 "X"
|
||||
#define PRIoLEAST32 "o"
|
||||
#define PRIuLEAST32 "u"
|
||||
#define PRIxLEAST32 "x"
|
||||
#define PRIXLEAST32 "X"
|
||||
#define PRIoFAST32 "o"
|
||||
#define PRIuFAST32 "u"
|
||||
#define PRIxFAST32 "x"
|
||||
#define PRIXFAST32 "X"
|
||||
|
||||
#define PRIo64 __PRI64_PREFIX "o"
|
||||
#define PRIu64 __PRI64_PREFIX "u"
|
||||
#define PRIx64 __PRI64_PREFIX "x"
|
||||
#define PRIX64 __PRI64_PREFIX "X"
|
||||
#define PRIoLEAST64 __PRI64_PREFIX "o"
|
||||
#define PRIuLEAST64 __PRI64_PREFIX "u"
|
||||
#define PRIxLEAST64 __PRI64_PREFIX "x"
|
||||
#define PRIXLEAST64 __PRI64_PREFIX "X"
|
||||
#define PRIoFAST64 __PRI64_PREFIX "o"
|
||||
#define PRIuFAST64 __PRI64_PREFIX "u"
|
||||
#define PRIxFAST64 __PRI64_PREFIX "x"
|
||||
#define PRIXFAST64 __PRI64_PREFIX "X"
|
||||
|
||||
#define PRIoMAX __PRI64_PREFIX "o"
|
||||
#define PRIuMAX __PRI64_PREFIX "u"
|
||||
#define PRIxMAX __PRI64_PREFIX "x"
|
||||
#define PRIXMAX __PRI64_PREFIX "X"
|
||||
|
||||
#define PRIoPTR __PRIPTR_PREFIX "o"
|
||||
#define PRIuPTR __PRIPTR_PREFIX "u"
|
||||
#define PRIxPTR __PRIPTR_PREFIX "x"
|
||||
#define PRIXPTR __PRIPTR_PREFIX "X"
|
||||
|
||||
// The fscanf macros for signed integers are:
|
||||
#define SCNd8 "d"
|
||||
#define SCNi8 "i"
|
||||
#define SCNdLEAST8 "d"
|
||||
#define SCNiLEAST8 "i"
|
||||
#define SCNdFAST8 "d"
|
||||
#define SCNiFAST8 "i"
|
||||
|
||||
#define SCNd16 "hd"
|
||||
#define SCNi16 "hi"
|
||||
#define SCNdLEAST16 "hd"
|
||||
#define SCNiLEAST16 "hi"
|
||||
#define SCNdFAST16 "hd"
|
||||
#define SCNiFAST16 "hi"
|
||||
|
||||
#define SCNd32 "ld"
|
||||
#define SCNi32 "li"
|
||||
#define SCNdLEAST32 "ld"
|
||||
#define SCNiLEAST32 "li"
|
||||
#define SCNdFAST32 "ld"
|
||||
#define SCNiFAST32 "li"
|
||||
|
||||
#define SCNd64 "I64d"
|
||||
#define SCNi64 "I64i"
|
||||
#define SCNdLEAST64 "I64d"
|
||||
#define SCNiLEAST64 "I64i"
|
||||
#define SCNdFAST64 "I64d"
|
||||
#define SCNiFAST64 "I64i"
|
||||
|
||||
#define SCNdMAX "I64d"
|
||||
#define SCNiMAX "I64i"
|
||||
|
||||
#ifdef _WIN64 // [
|
||||
# define SCNdPTR "I64d"
|
||||
# define SCNiPTR "I64i"
|
||||
#else // _WIN64 ][
|
||||
# define SCNdPTR "ld"
|
||||
# define SCNiPTR "li"
|
||||
#endif // _WIN64 ]
|
||||
|
||||
// The fscanf macros for unsigned integers are:
|
||||
#define SCNo8 "o"
|
||||
#define SCNu8 "u"
|
||||
#define SCNx8 "x"
|
||||
#define SCNX8 "X"
|
||||
#define SCNoLEAST8 "o"
|
||||
#define SCNuLEAST8 "u"
|
||||
#define SCNxLEAST8 "x"
|
||||
#define SCNXLEAST8 "X"
|
||||
#define SCNoFAST8 "o"
|
||||
#define SCNuFAST8 "u"
|
||||
#define SCNxFAST8 "x"
|
||||
#define SCNXFAST8 "X"
|
||||
|
||||
#define SCNo16 "ho"
|
||||
#define SCNu16 "hu"
|
||||
#define SCNx16 "hx"
|
||||
#define SCNX16 "hX"
|
||||
#define SCNoLEAST16 "ho"
|
||||
#define SCNuLEAST16 "hu"
|
||||
#define SCNxLEAST16 "hx"
|
||||
#define SCNXLEAST16 "hX"
|
||||
#define SCNoFAST16 "ho"
|
||||
#define SCNuFAST16 "hu"
|
||||
#define SCNxFAST16 "hx"
|
||||
#define SCNXFAST16 "hX"
|
||||
|
||||
#define SCNo32 "lo"
|
||||
#define SCNu32 "lu"
|
||||
#define SCNx32 "lx"
|
||||
#define SCNX32 "lX"
|
||||
#define SCNoLEAST32 "lo"
|
||||
#define SCNuLEAST32 "lu"
|
||||
#define SCNxLEAST32 "lx"
|
||||
#define SCNXLEAST32 "lX"
|
||||
#define SCNoFAST32 "lo"
|
||||
#define SCNuFAST32 "lu"
|
||||
#define SCNxFAST32 "lx"
|
||||
#define SCNXFAST32 "lX"
|
||||
|
||||
#define SCNo64 "I64o"
|
||||
#define SCNu64 "I64u"
|
||||
#define SCNx64 "I64x"
|
||||
#define SCNX64 "I64X"
|
||||
#define SCNoLEAST64 "I64o"
|
||||
#define SCNuLEAST64 "I64u"
|
||||
#define SCNxLEAST64 "I64x"
|
||||
#define SCNXLEAST64 "I64X"
|
||||
#define SCNoFAST64 "I64o"
|
||||
#define SCNuFAST64 "I64u"
|
||||
#define SCNxFAST64 "I64x"
|
||||
#define SCNXFAST64 "I64X"
|
||||
|
||||
#define SCNoMAX "I64o"
|
||||
#define SCNuMAX "I64u"
|
||||
#define SCNxMAX "I64x"
|
||||
#define SCNXMAX "I64X"
|
||||
|
||||
#ifdef _WIN64 // [
|
||||
# define SCNoPTR "I64o"
|
||||
# define SCNuPTR "I64u"
|
||||
# define SCNxPTR "I64x"
|
||||
# define SCNXPTR "I64X"
|
||||
#else // _WIN64 ][
|
||||
# define SCNoPTR "lo"
|
||||
# define SCNuPTR "lu"
|
||||
# define SCNxPTR "lx"
|
||||
# define SCNXPTR "lX"
|
||||
#endif // _WIN64 ]
|
||||
|
||||
#endif // __STDC_FORMAT_MACROS ]
|
||||
|
||||
// 7.8.2 Functions for greatest-width integer types
|
||||
|
||||
// 7.8.2.1 The imaxabs function
|
||||
#define imaxabs _abs64
|
||||
|
||||
// 7.8.2.2 The imaxdiv function
|
||||
|
||||
// This is modified version of div() function from Microsoft's div.c found
|
||||
// in %MSVC.NET%\crt\src\div.c
|
||||
#ifdef STATIC_IMAXDIV // [
|
||||
static
|
||||
#else // STATIC_IMAXDIV ][
|
||||
_inline
|
||||
#endif // STATIC_IMAXDIV ]
|
||||
imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom)
|
||||
{
|
||||
imaxdiv_t result;
|
||||
|
||||
result.quot = numer / denom;
|
||||
result.rem = numer % denom;
|
||||
|
||||
if (numer < 0 && result.rem > 0) {
|
||||
// did division wrong; must fix up
|
||||
++result.quot;
|
||||
result.rem -= denom;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// 7.8.2.3 The strtoimax and strtoumax functions
|
||||
#define strtoimax _strtoi64
|
||||
#define strtoumax _strtoui64
|
||||
|
||||
// 7.8.2.4 The wcstoimax and wcstoumax functions
|
||||
#define wcstoimax _wcstoi64
|
||||
#define wcstoumax _wcstoui64
|
||||
|
||||
|
||||
#endif // _MSC_INTTYPES_H_ ]
|
||||
@@ -5,11 +5,7 @@
|
||||
|
||||
/* MSVC doesn't define _Bool or bool in C, but does have BOOL */
|
||||
/* Note this doesn't pass autoconf's test because (bool) 0.5 != true */
|
||||
/* Clang-cl uses MSVC headers, so needs msvc_compat, but has _Bool as
|
||||
* a built-in type. */
|
||||
#ifndef __clang__
|
||||
typedef BOOL _Bool;
|
||||
#endif
|
||||
|
||||
#define bool _Bool
|
||||
#define true 1
|
||||
@@ -3,9 +3,8 @@
|
||||
|
||||
/* MSVC doesn't define ffs/ffsl. This dummy strings.h header is provided
|
||||
* for both */
|
||||
#ifdef _MSC_VER
|
||||
# include <intrin.h>
|
||||
# pragma intrinsic(_BitScanForward)
|
||||
#include <intrin.h>
|
||||
#pragma intrinsic(_BitScanForward)
|
||||
static __forceinline int ffsl(long x)
|
||||
{
|
||||
unsigned long i;
|
||||
@@ -21,39 +20,4 @@ static __forceinline int ffs(int x)
|
||||
return (ffsl(x));
|
||||
}
|
||||
|
||||
# ifdef _M_X64
|
||||
# pragma intrinsic(_BitScanForward64)
|
||||
# endif
|
||||
|
||||
static __forceinline int ffsll(unsigned __int64 x)
|
||||
{
|
||||
unsigned long i;
|
||||
#ifdef _M_X64
|
||||
if (_BitScanForward64(&i, x))
|
||||
return (i + 1);
|
||||
return (0);
|
||||
#else
|
||||
// Fallback for 32-bit build where 64-bit version not available
|
||||
// assuming little endian
|
||||
union {
|
||||
unsigned __int64 ll;
|
||||
unsigned long l[2];
|
||||
} s;
|
||||
|
||||
s.ll = x;
|
||||
|
||||
if (_BitScanForward(&i, s.l[0]))
|
||||
return (i + 1);
|
||||
else if(_BitScanForward(&i, s.l[1]))
|
||||
return (i + 33);
|
||||
return (0);
|
||||
#endif
|
||||
}
|
||||
|
||||
#else
|
||||
# define ffsll(x) __builtin_ffsll(x)
|
||||
# define ffsl(x) __builtin_ffsl(x)
|
||||
# define ffs(x) __builtin_ffs(x)
|
||||
#endif
|
||||
|
||||
#endif /* strings_h */
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
#ifndef MSVC_COMPAT_WINDOWS_EXTRA_H
|
||||
#define MSVC_COMPAT_WINDOWS_EXTRA_H
|
||||
|
||||
#include <errno.h>
|
||||
|
||||
#endif /* MSVC_COMPAT_WINDOWS_EXTRA_H */
|
||||
@@ -1,12 +0,0 @@
|
||||
prefix=@prefix@
|
||||
exec_prefix=@exec_prefix@
|
||||
libdir=@libdir@
|
||||
includedir=@includedir@
|
||||
install_suffix=@install_suffix@
|
||||
|
||||
Name: jemalloc
|
||||
Description: A general purpose malloc(3) implementation that emphasizes fragmentation avoidance and scalable concurrency support.
|
||||
URL: http://jemalloc.net/
|
||||
Version: @jemalloc_version@
|
||||
Cflags: -I${includedir}
|
||||
Libs: -L${libdir} -ljemalloc${install_suffix}
|
||||
@@ -1,24 +0,0 @@
|
||||
|
||||
How to build jemalloc for Windows
|
||||
=================================
|
||||
|
||||
1. Install Cygwin with at least the following packages:
|
||||
* autoconf
|
||||
* autogen
|
||||
* gawk
|
||||
* grep
|
||||
* sed
|
||||
|
||||
2. Install Visual Studio 2015 with Visual C++
|
||||
|
||||
3. Add Cygwin\bin to the PATH environment variable
|
||||
|
||||
4. Open "VS2015 x86 Native Tools Command Prompt"
|
||||
(note: x86/x64 doesn't matter at this point)
|
||||
|
||||
5. Generate header files:
|
||||
sh -c "CC=cl ./autogen.sh"
|
||||
|
||||
6. Now the project can be opened and built in Visual Studio:
|
||||
msvc\jemalloc_vc2015.sln
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 14
|
||||
VisualStudioVersion = 14.0.24720.0
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{70A99006-6DE9-472B-8F83-4CEE6C616DF3}"
|
||||
ProjectSection(SolutionItems) = preProject
|
||||
ReadMe.txt = ReadMe.txt
|
||||
EndProjectSection
|
||||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "jemalloc", "projects\vc2015\jemalloc\jemalloc.vcxproj", "{8D6BB292-9E1C-413D-9F98-4864BDC1514A}"
|
||||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_threads", "projects\vc2015\test_threads\test_threads.vcxproj", "{09028CFD-4EB7-491D-869C-0708DB97ED44}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|x64 = Debug|x64
|
||||
Debug|x86 = Debug|x86
|
||||
Debug-static|x64 = Debug-static|x64
|
||||
Debug-static|x86 = Debug-static|x86
|
||||
Release|x64 = Release|x64
|
||||
Release|x86 = Release|x86
|
||||
Release-static|x64 = Release-static|x64
|
||||
Release-static|x86 = Release-static|x86
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug|x64.Build.0 = Debug|x64
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug|x86.ActiveCfg = Debug|Win32
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug|x86.Build.0 = Debug|Win32
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug-static|x64.ActiveCfg = Debug-static|x64
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug-static|x64.Build.0 = Debug-static|x64
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug-static|x86.ActiveCfg = Debug-static|Win32
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug-static|x86.Build.0 = Debug-static|Win32
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release|x64.ActiveCfg = Release|x64
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release|x64.Build.0 = Release|x64
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release|x86.ActiveCfg = Release|Win32
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release|x86.Build.0 = Release|Win32
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release-static|x64.ActiveCfg = Release-static|x64
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release-static|x64.Build.0 = Release-static|x64
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release-static|x86.ActiveCfg = Release-static|Win32
|
||||
{8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release-static|x86.Build.0 = Release-static|Win32
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug|x64.Build.0 = Debug|x64
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug|x86.ActiveCfg = Debug|Win32
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug|x86.Build.0 = Debug|Win32
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug-static|x64.ActiveCfg = Debug-static|x64
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug-static|x64.Build.0 = Debug-static|x64
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug-static|x86.ActiveCfg = Debug-static|Win32
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug-static|x86.Build.0 = Debug-static|Win32
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Release|x64.ActiveCfg = Release|x64
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Release|x64.Build.0 = Release|x64
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Release|x86.ActiveCfg = Release|Win32
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Release|x86.Build.0 = Release|Win32
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Release-static|x64.ActiveCfg = Release-static|x64
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Release-static|x64.Build.0 = Release-static|x64
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Release-static|x86.ActiveCfg = Release-static|Win32
|
||||
{09028CFD-4EB7-491D-869C-0708DB97ED44}.Release-static|x86.Build.0 = Release-static|Win32
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
@@ -1,402 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug-static|Win32">
|
||||
<Configuration>Debug-static</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Debug-static|x64">
|
||||
<Configuration>Debug-static</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Debug|Win32">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release-static|Win32">
|
||||
<Configuration>Release-static</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release-static|x64">
|
||||
<Configuration>Release-static</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|Win32">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|x64">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\arena.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\assert.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\atomic.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\base.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\bitmap.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\chunk.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\chunk_dss.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\chunk_mmap.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ckh.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ctl.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\extent.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\hash.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\huge.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_decls.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_defs.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_macros.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mb.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mutex.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\nstime.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\pages.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ph.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\private_namespace.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\private_unnamespace.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\prng.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\prof.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\public_namespace.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\public_unnamespace.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ql.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\qr.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\quarantine.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\rb.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\rtree.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\size_classes.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\smoothstep.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\spin.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\stats.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\tcache.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ticker.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\tsd.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\util.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\witness.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_defs.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_macros.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_mangle.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_protos.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_protos_jet.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_rename.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_typedefs.h" />
|
||||
<ClInclude Include="..\..\..\..\include\msvc_compat\C99\stdbool.h" />
|
||||
<ClInclude Include="..\..\..\..\include\msvc_compat\C99\stdint.h" />
|
||||
<ClInclude Include="..\..\..\..\include\msvc_compat\strings.h" />
|
||||
<ClInclude Include="..\..\..\..\include\msvc_compat\windows_extra.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\..\src\arena.c" />
|
||||
<ClCompile Include="..\..\..\..\src\atomic.c" />
|
||||
<ClCompile Include="..\..\..\..\src\base.c" />
|
||||
<ClCompile Include="..\..\..\..\src\bitmap.c" />
|
||||
<ClCompile Include="..\..\..\..\src\chunk.c" />
|
||||
<ClCompile Include="..\..\..\..\src\chunk_dss.c" />
|
||||
<ClCompile Include="..\..\..\..\src\chunk_mmap.c" />
|
||||
<ClCompile Include="..\..\..\..\src\ckh.c" />
|
||||
<ClCompile Include="..\..\..\..\src\ctl.c" />
|
||||
<ClCompile Include="..\..\..\..\src\extent.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hash.c" />
|
||||
<ClCompile Include="..\..\..\..\src\huge.c" />
|
||||
<ClCompile Include="..\..\..\..\src\jemalloc.c" />
|
||||
<ClCompile Include="..\..\..\..\src\mb.c" />
|
||||
<ClCompile Include="..\..\..\..\src\mutex.c" />
|
||||
<ClCompile Include="..\..\..\..\src\nstime.c" />
|
||||
<ClCompile Include="..\..\..\..\src\pages.c" />
|
||||
<ClCompile Include="..\..\..\..\src\prng.c" />
|
||||
<ClCompile Include="..\..\..\..\src\prof.c" />
|
||||
<ClCompile Include="..\..\..\..\src\quarantine.c" />
|
||||
<ClCompile Include="..\..\..\..\src\rtree.c" />
|
||||
<ClCompile Include="..\..\..\..\src\spin.c" />
|
||||
<ClCompile Include="..\..\..\..\src\stats.c" />
|
||||
<ClCompile Include="..\..\..\..\src\tcache.c" />
|
||||
<ClCompile Include="..\..\..\..\src\ticker.c" />
|
||||
<ClCompile Include="..\..\..\..\src\tsd.c" />
|
||||
<ClCompile Include="..\..\..\..\src\util.c" />
|
||||
<ClCompile Include="..\..\..\..\src\witness.c" />
|
||||
</ItemGroup>
|
||||
<PropertyGroup Label="Globals">
|
||||
<ProjectGuid>{8D6BB292-9E1C-413D-9F98-4864BDC1514A}</ProjectGuid>
|
||||
<Keyword>Win32Proj</Keyword>
|
||||
<RootNamespace>jemalloc</RootNamespace>
|
||||
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|Win32'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|Win32'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="Shared">
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|Win32'" Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|Win32'" Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|x64'" Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|x64'" Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<PropertyGroup Label="UserMacros" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<TargetName>$(ProjectName)d</TargetName>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|Win32'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<TargetName>$(ProjectName)-$(PlatformToolset)-$(Configuration)</TargetName>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|Win32'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<TargetName>$(ProjectName)-$(PlatformToolset)-$(Configuration)</TargetName>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<TargetName>$(ProjectName)d</TargetName>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|x64'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<TargetName>$(ProjectName)-vc$(PlatformToolsetVersion)-$(Configuration)</TargetName>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|x64'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<TargetName>$(ProjectName)-vc$(PlatformToolsetVersion)-$(Configuration)</TargetName>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>_REENTRANT;_WINDLL;DLLEXPORT;JEMALLOC_DEBUG;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<DisableSpecificWarnings>4090;4146;4267;4334</DisableSpecificWarnings>
|
||||
<ProgramDataBaseFileName>$(OutputPath)$(TargetName).pdb</ProgramDataBaseFileName>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Windows</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|Win32'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>JEMALLOC_DEBUG;_REENTRANT;JEMALLOC_EXPORT=;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
|
||||
<DisableSpecificWarnings>4090;4146;4267;4334</DisableSpecificWarnings>
|
||||
<ProgramDataBaseFileName>$(OutputPath)$(TargetName).pdb</ProgramDataBaseFileName>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Windows</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>_REENTRANT;_WINDLL;DLLEXPORT;JEMALLOC_DEBUG;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<DisableSpecificWarnings>4090;4146;4267;4334</DisableSpecificWarnings>
|
||||
<ProgramDataBaseFileName>$(OutputPath)$(TargetName).pdb</ProgramDataBaseFileName>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Windows</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|x64'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>JEMALLOC_DEBUG;_REENTRANT;JEMALLOC_EXPORT=;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
|
||||
<DisableSpecificWarnings>4090;4146;4267;4334</DisableSpecificWarnings>
|
||||
<DebugInformationFormat>OldStyle</DebugInformationFormat>
|
||||
<MinimalRebuild>false</MinimalRebuild>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Windows</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>_REENTRANT;_WINDLL;DLLEXPORT;NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<DisableSpecificWarnings>4090;4146;4267;4334</DisableSpecificWarnings>
|
||||
<ProgramDataBaseFileName>$(OutputPath)$(TargetName).pdb</ProgramDataBaseFileName>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Windows</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|Win32'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>_REENTRANT;JEMALLOC_EXPORT=;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DisableSpecificWarnings>4090;4146;4267;4334</DisableSpecificWarnings>
|
||||
<ProgramDataBaseFileName>$(OutputPath)$(TargetName).pdb</ProgramDataBaseFileName>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Windows</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<PreprocessorDefinitions>_REENTRANT;_WINDLL;DLLEXPORT;NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<DisableSpecificWarnings>4090;4146;4267;4334</DisableSpecificWarnings>
|
||||
<ProgramDataBaseFileName>$(OutputPath)$(TargetName).pdb</ProgramDataBaseFileName>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Windows</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|x64'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>_REENTRANT;JEMALLOC_EXPORT=;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<DisableSpecificWarnings>4090;4146;4267;4334</DisableSpecificWarnings>
|
||||
<DebugInformationFormat>OldStyle</DebugInformationFormat>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Windows</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
</ImportGroup>
|
||||
</Project>
|
||||
@@ -1,272 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup>
|
||||
<Filter Include="Source Files">
|
||||
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
|
||||
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
|
||||
</Filter>
|
||||
<Filter Include="Header Files">
|
||||
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
|
||||
<Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
|
||||
</Filter>
|
||||
<Filter Include="Header Files\internal">
|
||||
<UniqueIdentifier>{5697dfa3-16cf-4932-b428-6e0ec6e9f98e}</UniqueIdentifier>
|
||||
</Filter>
|
||||
<Filter Include="Header Files\msvc_compat">
|
||||
<UniqueIdentifier>{0cbd2ca6-42a7-4f82-8517-d7e7a14fd986}</UniqueIdentifier>
|
||||
</Filter>
|
||||
<Filter Include="Header Files\msvc_compat\C99">
|
||||
<UniqueIdentifier>{0abe6f30-49b5-46dd-8aca-6e33363fa52c}</UniqueIdentifier>
|
||||
</Filter>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_defs.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_macros.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_mangle.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_protos.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_protos_jet.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_rename.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\jemalloc_typedefs.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\arena.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\assert.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\atomic.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\base.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\bitmap.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\chunk.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\chunk_dss.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\chunk_mmap.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ckh.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ctl.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\extent.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\hash.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\huge.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_decls.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_defs.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_macros.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mb.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mutex.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\nstime.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\pages.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ph.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\private_namespace.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\private_unnamespace.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\prng.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\prof.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\public_namespace.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\public_unnamespace.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ql.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\qr.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\quarantine.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\rb.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\rtree.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\size_classes.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\smoothstep.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\spin.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\stats.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\tcache.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ticker.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\tsd.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\util.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\witness.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\msvc_compat\strings.h">
|
||||
<Filter>Header Files\msvc_compat</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\msvc_compat\windows_extra.h">
|
||||
<Filter>Header Files\msvc_compat</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\msvc_compat\C99\stdbool.h">
|
||||
<Filter>Header Files\msvc_compat\C99</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\msvc_compat\C99\stdint.h">
|
||||
<Filter>Header Files\msvc_compat\C99</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\..\src\arena.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\atomic.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\base.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\bitmap.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\chunk.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\chunk_dss.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\chunk_mmap.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\ckh.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\ctl.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\extent.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\hash.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\huge.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\jemalloc.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\mb.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\mutex.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\nstime.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\pages.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\prng.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\prof.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\quarantine.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\rtree.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\spin.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\stats.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\tcache.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\ticker.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\tsd.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\util.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\witness.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,89 +0,0 @@
|
||||
// jemalloc C++ threaded test
|
||||
// Author: Rustam Abdullaev
|
||||
// Public Domain
|
||||
|
||||
#include <atomic>
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <random>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#include <stdio.h>
|
||||
#include <jemalloc/jemalloc.h>
|
||||
|
||||
using std::vector;
|
||||
using std::thread;
|
||||
using std::uniform_int_distribution;
|
||||
using std::minstd_rand;
|
||||
|
||||
int test_threads()
|
||||
{
|
||||
je_malloc_conf = "narenas:3";
|
||||
int narenas = 0;
|
||||
size_t sz = sizeof(narenas);
|
||||
je_mallctl("opt.narenas", (void *)&narenas, &sz, NULL, 0);
|
||||
if (narenas != 3) {
|
||||
printf("Error: unexpected number of arenas: %d\n", narenas);
|
||||
return 1;
|
||||
}
|
||||
static const int sizes[] = { 7, 16, 32, 60, 91, 100, 120, 144, 169, 199, 255, 400, 670, 900, 917, 1025, 3333, 5190, 13131, 49192, 99999, 123123, 255265, 2333111 };
|
||||
static const int numSizes = (int)(sizeof(sizes) / sizeof(sizes[0]));
|
||||
vector<thread> workers;
|
||||
static const int numThreads = narenas + 1, numAllocsMax = 25, numIter1 = 50, numIter2 = 50;
|
||||
je_malloc_stats_print(NULL, NULL, NULL);
|
||||
size_t allocated1;
|
||||
size_t sz1 = sizeof(allocated1);
|
||||
je_mallctl("stats.active", (void *)&allocated1, &sz1, NULL, 0);
|
||||
printf("\nPress Enter to start threads...\n");
|
||||
getchar();
|
||||
printf("Starting %d threads x %d x %d iterations...\n", numThreads, numIter1, numIter2);
|
||||
for (int i = 0; i < numThreads; i++) {
|
||||
workers.emplace_back([tid=i]() {
|
||||
uniform_int_distribution<int> sizeDist(0, numSizes - 1);
|
||||
minstd_rand rnd(tid * 17);
|
||||
uint8_t* ptrs[numAllocsMax];
|
||||
int ptrsz[numAllocsMax];
|
||||
for (int i = 0; i < numIter1; ++i) {
|
||||
thread t([&]() {
|
||||
for (int i = 0; i < numIter2; ++i) {
|
||||
const int numAllocs = numAllocsMax - sizeDist(rnd);
|
||||
for (int j = 0; j < numAllocs; j += 64) {
|
||||
const int x = sizeDist(rnd);
|
||||
const int sz = sizes[x];
|
||||
ptrsz[j] = sz;
|
||||
ptrs[j] = (uint8_t*)je_malloc(sz);
|
||||
if (!ptrs[j]) {
|
||||
printf("Unable to allocate %d bytes in thread %d, iter %d, alloc %d. %d\n", sz, tid, i, j, x);
|
||||
exit(1);
|
||||
}
|
||||
for (int k = 0; k < sz; k++)
|
||||
ptrs[j][k] = tid + k;
|
||||
}
|
||||
for (int j = 0; j < numAllocs; j += 64) {
|
||||
for (int k = 0, sz = ptrsz[j]; k < sz; k++)
|
||||
if (ptrs[j][k] != (uint8_t)(tid + k)) {
|
||||
printf("Memory error in thread %d, iter %d, alloc %d @ %d : %02X!=%02X\n", tid, i, j, k, ptrs[j][k], (uint8_t)(tid + k));
|
||||
exit(1);
|
||||
}
|
||||
je_free(ptrs[j]);
|
||||
}
|
||||
}
|
||||
});
|
||||
t.join();
|
||||
}
|
||||
});
|
||||
}
|
||||
for (thread& t : workers) {
|
||||
t.join();
|
||||
}
|
||||
je_malloc_stats_print(NULL, NULL, NULL);
|
||||
size_t allocated2;
|
||||
je_mallctl("stats.active", (void *)&allocated2, &sz1, NULL, 0);
|
||||
size_t leaked = allocated2 - allocated1;
|
||||
printf("\nDone. Leaked: %zd bytes\n", leaked);
|
||||
bool failed = leaked > 65536; // in case C++ runtime allocated something (e.g. iostream locale or facet)
|
||||
printf("\nTest %s!\n", (failed ? "FAILED" : "successful"));
|
||||
printf("\nPress Enter to continue...\n");
|
||||
getchar();
|
||||
return failed ? 1 : 0;
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
int test_threads();
|
||||
@@ -1,327 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug-static|Win32">
|
||||
<Configuration>Debug-static</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Debug-static|x64">
|
||||
<Configuration>Debug-static</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Debug|Win32">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release-static|Win32">
|
||||
<Configuration>Release-static</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release-static|x64">
|
||||
<Configuration>Release-static</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|Win32">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|x64">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
</ItemGroup>
|
||||
<PropertyGroup Label="Globals">
|
||||
<ProjectGuid>{09028CFD-4EB7-491D-869C-0708DB97ED44}</ProjectGuid>
|
||||
<Keyword>Win32Proj</Keyword>
|
||||
<RootNamespace>test_threads</RootNamespace>
|
||||
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="Shared">
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|Win32'" Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|Win32'" Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|x64'" Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|x64'" Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<PropertyGroup Label="UserMacros" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|Win32'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|Win32'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|x64'">
|
||||
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)</AdditionalLibraryDirectories>
|
||||
<AdditionalDependencies>jemallocd.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|Win32'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>JEMALLOC_EXPORT=;JEMALLOC_STATIC;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)</AdditionalLibraryDirectories>
|
||||
<AdditionalDependencies>jemalloc-$(PlatformToolset)-$(Configuration).lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<AdditionalDependencies>jemallocd.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)</AdditionalLibraryDirectories>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug-static|x64'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>JEMALLOC_EXPORT=;JEMALLOC_STATIC;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<AdditionalDependencies>jemalloc-vc$(PlatformToolsetVersion)-$(Configuration).lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)</AdditionalLibraryDirectories>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)</AdditionalLibraryDirectories>
|
||||
<AdditionalDependencies>jemalloc.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|Win32'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>JEMALLOC_EXPORT=;JEMALLOC_STATIC;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)</AdditionalLibraryDirectories>
|
||||
<AdditionalDependencies>jemalloc-$(PlatformToolset)-$(Configuration).lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)</AdditionalLibraryDirectories>
|
||||
<AdditionalDependencies>jemalloc.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release-static|x64'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>JEMALLOC_EXPORT=;JEMALLOC_STATIC;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<AdditionalIncludeDirectories>..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)</AdditionalLibraryDirectories>
|
||||
<AdditionalDependencies>jemalloc-vc$(PlatformToolsetVersion)-$(Configuration).lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="test_threads.cpp" />
|
||||
<ClCompile Include="test_threads_main.cpp" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\jemalloc\jemalloc.vcxproj">
|
||||
<Project>{8d6bb292-9e1c-413d-9f98-4864bdc1514a}</Project>
|
||||
</ProjectReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="test_threads.h" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
</ImportGroup>
|
||||
</Project>
|
||||
@@ -1,26 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup>
|
||||
<Filter Include="Source Files">
|
||||
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
|
||||
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
|
||||
</Filter>
|
||||
<Filter Include="Header Files">
|
||||
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
|
||||
<Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
|
||||
</Filter>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="test_threads.cpp">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="test_threads_main.cpp">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="test_threads.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,12 +0,0 @@
|
||||
#include "test_threads.h"
|
||||
#include <future>
|
||||
#include <functional>
|
||||
#include <chrono>
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
int rc = test_threads();
|
||||
return rc;
|
||||
}
|
||||
3877
src/arena.c
3877
src/arena.c
File diff suppressed because it is too large
Load Diff
217
src/base.c
217
src/base.c
@@ -5,183 +5,138 @@
|
||||
/* Data. */
|
||||
|
||||
static malloc_mutex_t base_mtx;
|
||||
static size_t base_extent_sn_next;
|
||||
static extent_tree_t base_avail_szsnad;
|
||||
|
||||
/*
|
||||
* Current pages that are being used for internal memory allocations. These
|
||||
* pages are carved up in cacheline-size quanta, so that there is no chance of
|
||||
* false cache line sharing.
|
||||
*/
|
||||
static void *base_pages;
|
||||
static void *base_next_addr;
|
||||
static void *base_past_addr; /* Addr immediately past base_pages. */
|
||||
static extent_node_t *base_nodes;
|
||||
static size_t base_allocated;
|
||||
static size_t base_resident;
|
||||
static size_t base_mapped;
|
||||
|
||||
/******************************************************************************/
|
||||
/* Function prototypes for non-inline static functions. */
|
||||
|
||||
static bool base_pages_alloc(size_t minsize);
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
static extent_node_t *
|
||||
base_node_try_alloc(tsdn_t *tsdn)
|
||||
static bool
|
||||
base_pages_alloc(size_t minsize)
|
||||
{
|
||||
extent_node_t *node;
|
||||
size_t csize;
|
||||
bool zero;
|
||||
|
||||
malloc_mutex_assert_owner(tsdn, &base_mtx);
|
||||
|
||||
if (base_nodes == NULL)
|
||||
return (NULL);
|
||||
node = base_nodes;
|
||||
base_nodes = *(extent_node_t **)node;
|
||||
JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
|
||||
return (node);
|
||||
}
|
||||
|
||||
static void
|
||||
base_node_dalloc(tsdn_t *tsdn, extent_node_t *node)
|
||||
{
|
||||
|
||||
malloc_mutex_assert_owner(tsdn, &base_mtx);
|
||||
|
||||
JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
|
||||
*(extent_node_t **)node = base_nodes;
|
||||
base_nodes = node;
|
||||
}
|
||||
|
||||
static void
|
||||
base_extent_node_init(extent_node_t *node, void *addr, size_t size)
|
||||
{
|
||||
size_t sn = atomic_add_z(&base_extent_sn_next, 1) - 1;
|
||||
|
||||
extent_node_init(node, NULL, addr, size, sn, true, true);
|
||||
}
|
||||
|
||||
static extent_node_t *
|
||||
base_chunk_alloc(tsdn_t *tsdn, size_t minsize)
|
||||
{
|
||||
extent_node_t *node;
|
||||
size_t csize, nsize;
|
||||
void *addr;
|
||||
|
||||
malloc_mutex_assert_owner(tsdn, &base_mtx);
|
||||
assert(minsize != 0);
|
||||
node = base_node_try_alloc(tsdn);
|
||||
/* Allocate enough space to also carve a node out if necessary. */
|
||||
nsize = (node == NULL) ? CACHELINE_CEILING(sizeof(extent_node_t)) : 0;
|
||||
csize = CHUNK_CEILING(minsize + nsize);
|
||||
addr = chunk_alloc_base(csize);
|
||||
if (addr == NULL) {
|
||||
if (node != NULL)
|
||||
base_node_dalloc(tsdn, node);
|
||||
return (NULL);
|
||||
}
|
||||
base_mapped += csize;
|
||||
if (node == NULL) {
|
||||
node = (extent_node_t *)addr;
|
||||
addr = (void *)((uintptr_t)addr + nsize);
|
||||
csize -= nsize;
|
||||
if (config_stats) {
|
||||
base_allocated += nsize;
|
||||
base_resident += PAGE_CEILING(nsize);
|
||||
}
|
||||
}
|
||||
base_extent_node_init(node, addr, csize);
|
||||
return (node);
|
||||
csize = CHUNK_CEILING(minsize);
|
||||
zero = false;
|
||||
base_pages = chunk_alloc(csize, chunksize, true, &zero,
|
||||
chunk_dss_prec_get());
|
||||
if (base_pages == NULL)
|
||||
return (true);
|
||||
base_next_addr = base_pages;
|
||||
base_past_addr = (void *)((uintptr_t)base_pages + csize);
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
/*
|
||||
* base_alloc() guarantees demand-zeroed memory, in order to make multi-page
|
||||
* sparse data structures such as radix tree nodes efficient with respect to
|
||||
* physical memory usage.
|
||||
*/
|
||||
void *
|
||||
base_alloc(tsdn_t *tsdn, size_t size)
|
||||
base_alloc(size_t size)
|
||||
{
|
||||
void *ret;
|
||||
size_t csize, usize;
|
||||
extent_node_t *node;
|
||||
extent_node_t key;
|
||||
size_t csize;
|
||||
|
||||
/*
|
||||
* Round size up to nearest multiple of the cacheline size, so that
|
||||
* there is no chance of false cache line sharing.
|
||||
*/
|
||||
/* Round size up to nearest multiple of the cacheline size. */
|
||||
csize = CACHELINE_CEILING(size);
|
||||
|
||||
usize = s2u(csize);
|
||||
extent_node_init(&key, NULL, NULL, usize, 0, false, false);
|
||||
malloc_mutex_lock(tsdn, &base_mtx);
|
||||
node = extent_tree_szsnad_nsearch(&base_avail_szsnad, &key);
|
||||
if (node != NULL) {
|
||||
/* Use existing space. */
|
||||
extent_tree_szsnad_remove(&base_avail_szsnad, node);
|
||||
} else {
|
||||
/* Try to allocate more space. */
|
||||
node = base_chunk_alloc(tsdn, csize);
|
||||
malloc_mutex_lock(&base_mtx);
|
||||
/* Make sure there's enough space for the allocation. */
|
||||
if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) {
|
||||
if (base_pages_alloc(csize)) {
|
||||
malloc_mutex_unlock(&base_mtx);
|
||||
return (NULL);
|
||||
}
|
||||
}
|
||||
if (node == NULL) {
|
||||
ret = NULL;
|
||||
goto label_return;
|
||||
/* Allocate. */
|
||||
ret = base_next_addr;
|
||||
base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
|
||||
malloc_mutex_unlock(&base_mtx);
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(ret, csize);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
void *
|
||||
base_calloc(size_t number, size_t size)
|
||||
{
|
||||
void *ret = base_alloc(number * size);
|
||||
|
||||
if (ret != NULL)
|
||||
memset(ret, 0, number * size);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
extent_node_t *
|
||||
base_node_alloc(void)
|
||||
{
|
||||
extent_node_t *ret;
|
||||
|
||||
malloc_mutex_lock(&base_mtx);
|
||||
if (base_nodes != NULL) {
|
||||
ret = base_nodes;
|
||||
base_nodes = *(extent_node_t **)ret;
|
||||
malloc_mutex_unlock(&base_mtx);
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(ret, sizeof(extent_node_t));
|
||||
} else {
|
||||
malloc_mutex_unlock(&base_mtx);
|
||||
ret = (extent_node_t *)base_alloc(sizeof(extent_node_t));
|
||||
}
|
||||
|
||||
ret = extent_node_addr_get(node);
|
||||
if (extent_node_size_get(node) > csize) {
|
||||
extent_node_addr_set(node, (void *)((uintptr_t)ret + csize));
|
||||
extent_node_size_set(node, extent_node_size_get(node) - csize);
|
||||
extent_tree_szsnad_insert(&base_avail_szsnad, node);
|
||||
} else
|
||||
base_node_dalloc(tsdn, node);
|
||||
if (config_stats) {
|
||||
base_allocated += csize;
|
||||
/*
|
||||
* Add one PAGE to base_resident for every page boundary that is
|
||||
* crossed by the new allocation.
|
||||
*/
|
||||
base_resident += PAGE_CEILING((uintptr_t)ret + csize) -
|
||||
PAGE_CEILING((uintptr_t)ret);
|
||||
}
|
||||
JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, csize);
|
||||
label_return:
|
||||
malloc_mutex_unlock(tsdn, &base_mtx);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
void
|
||||
base_stats_get(tsdn_t *tsdn, size_t *allocated, size_t *resident,
|
||||
size_t *mapped)
|
||||
base_node_dealloc(extent_node_t *node)
|
||||
{
|
||||
|
||||
malloc_mutex_lock(tsdn, &base_mtx);
|
||||
assert(base_allocated <= base_resident);
|
||||
assert(base_resident <= base_mapped);
|
||||
*allocated = base_allocated;
|
||||
*resident = base_resident;
|
||||
*mapped = base_mapped;
|
||||
malloc_mutex_unlock(tsdn, &base_mtx);
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
|
||||
malloc_mutex_lock(&base_mtx);
|
||||
*(extent_node_t **)node = base_nodes;
|
||||
base_nodes = node;
|
||||
malloc_mutex_unlock(&base_mtx);
|
||||
}
|
||||
|
||||
bool
|
||||
base_boot(void)
|
||||
{
|
||||
|
||||
if (malloc_mutex_init(&base_mtx, "base", WITNESS_RANK_BASE))
|
||||
return (true);
|
||||
base_extent_sn_next = 0;
|
||||
extent_tree_szsnad_new(&base_avail_szsnad);
|
||||
base_nodes = NULL;
|
||||
if (malloc_mutex_init(&base_mtx))
|
||||
return (true);
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
void
|
||||
base_prefork(tsdn_t *tsdn)
|
||||
base_prefork(void)
|
||||
{
|
||||
|
||||
malloc_mutex_prefork(tsdn, &base_mtx);
|
||||
malloc_mutex_prefork(&base_mtx);
|
||||
}
|
||||
|
||||
void
|
||||
base_postfork_parent(tsdn_t *tsdn)
|
||||
base_postfork_parent(void)
|
||||
{
|
||||
|
||||
malloc_mutex_postfork_parent(tsdn, &base_mtx);
|
||||
malloc_mutex_postfork_parent(&base_mtx);
|
||||
}
|
||||
|
||||
void
|
||||
base_postfork_child(tsdn_t *tsdn)
|
||||
base_postfork_child(void)
|
||||
{
|
||||
|
||||
malloc_mutex_postfork_child(tsdn, &base_mtx);
|
||||
malloc_mutex_postfork_child(&base_mtx);
|
||||
}
|
||||
|
||||
75
src/bitmap.c
75
src/bitmap.c
@@ -2,8 +2,19 @@
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
|
||||
/******************************************************************************/
|
||||
/* Function prototypes for non-inline static functions. */
|
||||
|
||||
#ifdef USE_TREE
|
||||
static size_t bits2groups(size_t nbits);
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
static size_t
|
||||
bits2groups(size_t nbits)
|
||||
{
|
||||
|
||||
return ((nbits >> LG_BITMAP_GROUP_NBITS) +
|
||||
!!(nbits & BITMAP_GROUP_NBITS_MASK));
|
||||
}
|
||||
|
||||
void
|
||||
bitmap_info_init(bitmap_info_t *binfo, size_t nbits)
|
||||
@@ -20,25 +31,33 @@ bitmap_info_init(bitmap_info_t *binfo, size_t nbits)
|
||||
* that requires only one group.
|
||||
*/
|
||||
binfo->levels[0].group_offset = 0;
|
||||
group_count = BITMAP_BITS2GROUPS(nbits);
|
||||
group_count = bits2groups(nbits);
|
||||
for (i = 1; group_count > 1; i++) {
|
||||
assert(i < BITMAP_MAX_LEVELS);
|
||||
binfo->levels[i].group_offset = binfo->levels[i-1].group_offset
|
||||
+ group_count;
|
||||
group_count = BITMAP_BITS2GROUPS(group_count);
|
||||
group_count = bits2groups(group_count);
|
||||
}
|
||||
binfo->levels[i].group_offset = binfo->levels[i-1].group_offset
|
||||
+ group_count;
|
||||
assert(binfo->levels[i].group_offset <= BITMAP_GROUPS_MAX);
|
||||
binfo->nlevels = i;
|
||||
binfo->nbits = nbits;
|
||||
}
|
||||
|
||||
static size_t
|
||||
size_t
|
||||
bitmap_info_ngroups(const bitmap_info_t *binfo)
|
||||
{
|
||||
|
||||
return (binfo->levels[binfo->nlevels].group_offset);
|
||||
return (binfo->levels[binfo->nlevels].group_offset << LG_SIZEOF_BITMAP);
|
||||
}
|
||||
|
||||
size_t
|
||||
bitmap_size(size_t nbits)
|
||||
{
|
||||
bitmap_info_t binfo;
|
||||
|
||||
bitmap_info_init(&binfo, nbits);
|
||||
return (bitmap_info_ngroups(&binfo));
|
||||
}
|
||||
|
||||
void
|
||||
@@ -54,7 +73,8 @@ bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo)
|
||||
* correspond to the first logical bit in the group, so extra bits
|
||||
* are the most significant bits of the last group.
|
||||
*/
|
||||
memset(bitmap, 0xffU, bitmap_size(binfo));
|
||||
memset(bitmap, 0xffU, binfo->levels[binfo->nlevels].group_offset <<
|
||||
LG_SIZEOF_BITMAP);
|
||||
extra = (BITMAP_GROUP_NBITS - (binfo->nbits & BITMAP_GROUP_NBITS_MASK))
|
||||
& BITMAP_GROUP_NBITS_MASK;
|
||||
if (extra != 0)
|
||||
@@ -68,44 +88,3 @@ bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo)
|
||||
bitmap[binfo->levels[i+1].group_offset - 1] >>= extra;
|
||||
}
|
||||
}
|
||||
|
||||
#else /* USE_TREE */
|
||||
|
||||
void
|
||||
bitmap_info_init(bitmap_info_t *binfo, size_t nbits)
|
||||
{
|
||||
|
||||
assert(nbits > 0);
|
||||
assert(nbits <= (ZU(1) << LG_BITMAP_MAXBITS));
|
||||
|
||||
binfo->ngroups = BITMAP_BITS2GROUPS(nbits);
|
||||
binfo->nbits = nbits;
|
||||
}
|
||||
|
||||
static size_t
|
||||
bitmap_info_ngroups(const bitmap_info_t *binfo)
|
||||
{
|
||||
|
||||
return (binfo->ngroups);
|
||||
}
|
||||
|
||||
void
|
||||
bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo)
|
||||
{
|
||||
size_t extra;
|
||||
|
||||
memset(bitmap, 0xffU, bitmap_size(binfo));
|
||||
extra = (BITMAP_GROUP_NBITS - (binfo->nbits & BITMAP_GROUP_NBITS_MASK))
|
||||
& BITMAP_GROUP_NBITS_MASK;
|
||||
if (extra != 0)
|
||||
bitmap[binfo->ngroups - 1] >>= extra;
|
||||
}
|
||||
|
||||
#endif /* USE_TREE */
|
||||
|
||||
size_t
|
||||
bitmap_size(const bitmap_info_t *binfo)
|
||||
{
|
||||
|
||||
return (bitmap_info_ngroups(binfo) << LG_SIZEOF_BITMAP);
|
||||
}
|
||||
|
||||
908
src/chunk.c
908
src/chunk.c
File diff suppressed because it is too large
Load Diff
214
src/chunk_dss.c
214
src/chunk_dss.c
@@ -10,19 +10,20 @@ const char *dss_prec_names[] = {
|
||||
"N/A"
|
||||
};
|
||||
|
||||
/* Current dss precedence default, used when creating new arenas. */
|
||||
static dss_prec_t dss_prec_default = DSS_PREC_DEFAULT;
|
||||
|
||||
/*
|
||||
* Current dss precedence default, used when creating new arenas. NB: This is
|
||||
* stored as unsigned rather than dss_prec_t because in principle there's no
|
||||
* guarantee that sizeof(dss_prec_t) is the same as sizeof(unsigned), and we use
|
||||
* atomic operations to synchronize the setting.
|
||||
* Protects sbrk() calls. This avoids malloc races among threads, though it
|
||||
* does not protect against races with threads that call sbrk() directly.
|
||||
*/
|
||||
static unsigned dss_prec_default = (unsigned)DSS_PREC_DEFAULT;
|
||||
static malloc_mutex_t dss_mtx;
|
||||
|
||||
/* Base address of the DSS. */
|
||||
static void *dss_base;
|
||||
/* Atomic boolean indicating whether the DSS is exhausted. */
|
||||
static unsigned dss_exhausted;
|
||||
/* Atomic current upper limit on DSS addresses. */
|
||||
/* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */
|
||||
static void *dss_prev;
|
||||
/* Current upper limit on DSS addresses. */
|
||||
static void *dss_max;
|
||||
|
||||
/******************************************************************************/
|
||||
@@ -31,7 +32,7 @@ static void *
|
||||
chunk_dss_sbrk(intptr_t increment)
|
||||
{
|
||||
|
||||
#ifdef JEMALLOC_DSS
|
||||
#ifdef JEMALLOC_HAVE_SBRK
|
||||
return (sbrk(increment));
|
||||
#else
|
||||
not_implemented();
|
||||
@@ -44,9 +45,11 @@ chunk_dss_prec_get(void)
|
||||
{
|
||||
dss_prec_t ret;
|
||||
|
||||
if (!have_dss)
|
||||
if (config_dss == false)
|
||||
return (dss_prec_disabled);
|
||||
ret = (dss_prec_t)atomic_read_u(&dss_prec_default);
|
||||
malloc_mutex_lock(&dss_mtx);
|
||||
ret = dss_prec_default;
|
||||
malloc_mutex_unlock(&dss_mtx);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
@@ -54,50 +57,20 @@ bool
|
||||
chunk_dss_prec_set(dss_prec_t dss_prec)
|
||||
{
|
||||
|
||||
if (!have_dss)
|
||||
return (dss_prec != dss_prec_disabled);
|
||||
atomic_write_u(&dss_prec_default, (unsigned)dss_prec);
|
||||
if (config_dss == false)
|
||||
return (true);
|
||||
malloc_mutex_lock(&dss_mtx);
|
||||
dss_prec_default = dss_prec;
|
||||
malloc_mutex_unlock(&dss_mtx);
|
||||
return (false);
|
||||
}
|
||||
|
||||
static void *
|
||||
chunk_dss_max_update(void *new_addr)
|
||||
{
|
||||
void *max_cur;
|
||||
spin_t spinner;
|
||||
|
||||
/*
|
||||
* Get the current end of the DSS as max_cur and assure that dss_max is
|
||||
* up to date.
|
||||
*/
|
||||
spin_init(&spinner);
|
||||
while (true) {
|
||||
void *max_prev = atomic_read_p(&dss_max);
|
||||
|
||||
max_cur = chunk_dss_sbrk(0);
|
||||
if ((uintptr_t)max_prev > (uintptr_t)max_cur) {
|
||||
/*
|
||||
* Another thread optimistically updated dss_max. Wait
|
||||
* for it to finish.
|
||||
*/
|
||||
spin_adaptive(&spinner);
|
||||
continue;
|
||||
}
|
||||
if (!atomic_cas_p(&dss_max, max_prev, max_cur))
|
||||
break;
|
||||
}
|
||||
/* Fixed new_addr can only be supported if it is at the edge of DSS. */
|
||||
if (new_addr != NULL && max_cur != new_addr)
|
||||
return (NULL);
|
||||
|
||||
return (max_cur);
|
||||
}
|
||||
|
||||
void *
|
||||
chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
||||
size_t alignment, bool *zero, bool *commit)
|
||||
chunk_alloc_dss(size_t size, size_t alignment, bool *zero)
|
||||
{
|
||||
cassert(have_dss);
|
||||
void *ret;
|
||||
|
||||
cassert(config_dss);
|
||||
assert(size > 0 && (size & chunksize_mask) == 0);
|
||||
assert(alignment > 0 && (alignment & chunksize_mask) == 0);
|
||||
|
||||
@@ -108,21 +81,20 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
||||
if ((intptr_t)size < 0)
|
||||
return (NULL);
|
||||
|
||||
if (!atomic_read_u(&dss_exhausted)) {
|
||||
malloc_mutex_lock(&dss_mtx);
|
||||
if (dss_prev != (void *)-1) {
|
||||
size_t gap_size, cpad_size;
|
||||
void *cpad, *dss_next;
|
||||
intptr_t incr;
|
||||
|
||||
/*
|
||||
* The loop is necessary to recover from races with other
|
||||
* threads that are using the DSS for something other than
|
||||
* malloc.
|
||||
*/
|
||||
while (true) {
|
||||
void *ret, *cpad, *max_cur, *dss_next, *dss_prev;
|
||||
size_t gap_size, cpad_size;
|
||||
intptr_t incr;
|
||||
|
||||
max_cur = chunk_dss_max_update(new_addr);
|
||||
if (max_cur == NULL)
|
||||
goto label_oom;
|
||||
|
||||
do {
|
||||
/* Get the current end of the DSS. */
|
||||
dss_max = chunk_dss_sbrk(0);
|
||||
/*
|
||||
* Calculate how much padding is necessary to
|
||||
* chunk-align the end of the DSS.
|
||||
@@ -140,99 +112,87 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
||||
cpad_size = (uintptr_t)ret - (uintptr_t)cpad;
|
||||
dss_next = (void *)((uintptr_t)ret + size);
|
||||
if ((uintptr_t)ret < (uintptr_t)dss_max ||
|
||||
(uintptr_t)dss_next < (uintptr_t)dss_max)
|
||||
goto label_oom; /* Wrap-around. */
|
||||
(uintptr_t)dss_next < (uintptr_t)dss_max) {
|
||||
/* Wrap-around. */
|
||||
malloc_mutex_unlock(&dss_mtx);
|
||||
return (NULL);
|
||||
}
|
||||
incr = gap_size + cpad_size + size;
|
||||
|
||||
/*
|
||||
* Optimistically update dss_max, and roll back below if
|
||||
* sbrk() fails. No other thread will try to extend the
|
||||
* DSS while dss_max is greater than the current DSS
|
||||
* max reported by sbrk(0).
|
||||
*/
|
||||
if (atomic_cas_p(&dss_max, max_cur, dss_next))
|
||||
continue;
|
||||
|
||||
/* Try to allocate. */
|
||||
dss_prev = chunk_dss_sbrk(incr);
|
||||
if (dss_prev == max_cur) {
|
||||
if (dss_prev == dss_max) {
|
||||
/* Success. */
|
||||
if (cpad_size != 0) {
|
||||
chunk_hooks_t chunk_hooks =
|
||||
CHUNK_HOOKS_INITIALIZER;
|
||||
chunk_dalloc_wrapper(tsdn, arena,
|
||||
&chunk_hooks, cpad, cpad_size,
|
||||
arena_extent_sn_next(arena), false,
|
||||
true);
|
||||
}
|
||||
dss_max = dss_next;
|
||||
malloc_mutex_unlock(&dss_mtx);
|
||||
if (cpad_size != 0)
|
||||
chunk_unmap(cpad, cpad_size);
|
||||
if (*zero) {
|
||||
JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(
|
||||
ret, size);
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
if (!*commit)
|
||||
*commit = pages_decommit(ret, size);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
/*
|
||||
* Failure, whether due to OOM or a race with a raw
|
||||
* sbrk() call from outside the allocator. Try to roll
|
||||
* back optimistic dss_max update; if rollback fails,
|
||||
* it's due to another caller of this function having
|
||||
* succeeded since this invocation started, in which
|
||||
* case rollback is not necessary.
|
||||
*/
|
||||
atomic_cas_p(&dss_max, dss_next, max_cur);
|
||||
if (dss_prev == (void *)-1) {
|
||||
/* OOM. */
|
||||
atomic_write_u(&dss_exhausted, (unsigned)true);
|
||||
goto label_oom;
|
||||
}
|
||||
}
|
||||
} while (dss_prev != (void *)-1);
|
||||
}
|
||||
label_oom:
|
||||
malloc_mutex_unlock(&dss_mtx);
|
||||
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static bool
|
||||
chunk_in_dss_helper(void *chunk, void *max)
|
||||
{
|
||||
|
||||
return ((uintptr_t)chunk >= (uintptr_t)dss_base && (uintptr_t)chunk <
|
||||
(uintptr_t)max);
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_in_dss(void *chunk)
|
||||
{
|
||||
bool ret;
|
||||
|
||||
cassert(have_dss);
|
||||
cassert(config_dss);
|
||||
|
||||
return (chunk_in_dss_helper(chunk, atomic_read_p(&dss_max)));
|
||||
malloc_mutex_lock(&dss_mtx);
|
||||
if ((uintptr_t)chunk >= (uintptr_t)dss_base
|
||||
&& (uintptr_t)chunk < (uintptr_t)dss_max)
|
||||
ret = true;
|
||||
else
|
||||
ret = false;
|
||||
malloc_mutex_unlock(&dss_mtx);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_dss_mergeable(void *chunk_a, void *chunk_b)
|
||||
{
|
||||
void *max;
|
||||
|
||||
cassert(have_dss);
|
||||
|
||||
max = atomic_read_p(&dss_max);
|
||||
return (chunk_in_dss_helper(chunk_a, max) ==
|
||||
chunk_in_dss_helper(chunk_b, max));
|
||||
}
|
||||
|
||||
void
|
||||
chunk_dss_boot(void)
|
||||
{
|
||||
|
||||
cassert(have_dss);
|
||||
cassert(config_dss);
|
||||
|
||||
if (malloc_mutex_init(&dss_mtx))
|
||||
return (true);
|
||||
dss_base = chunk_dss_sbrk(0);
|
||||
dss_exhausted = (unsigned)(dss_base == (void *)-1);
|
||||
dss_prev = dss_base;
|
||||
dss_max = dss_base;
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
void
|
||||
chunk_dss_prefork(void)
|
||||
{
|
||||
|
||||
if (config_dss)
|
||||
malloc_mutex_prefork(&dss_mtx);
|
||||
}
|
||||
|
||||
void
|
||||
chunk_dss_postfork_parent(void)
|
||||
{
|
||||
|
||||
if (config_dss)
|
||||
malloc_mutex_postfork_parent(&dss_mtx);
|
||||
}
|
||||
|
||||
void
|
||||
chunk_dss_postfork_child(void)
|
||||
{
|
||||
|
||||
if (config_dss)
|
||||
malloc_mutex_postfork_child(&dss_mtx);
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
162
src/chunk_mmap.c
162
src/chunk_mmap.c
@@ -1,27 +1,161 @@
|
||||
#define JEMALLOC_CHUNK_MMAP_C_
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
|
||||
/******************************************************************************/
|
||||
/* Function prototypes for non-inline static functions. */
|
||||
|
||||
static void *pages_map(void *addr, size_t size);
|
||||
static void pages_unmap(void *addr, size_t size);
|
||||
static void *chunk_alloc_mmap_slow(size_t size, size_t alignment,
|
||||
bool *zero);
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
static void *
|
||||
chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero, bool *commit)
|
||||
pages_map(void *addr, size_t size)
|
||||
{
|
||||
void *ret;
|
||||
size_t alloc_size;
|
||||
|
||||
assert(size != 0);
|
||||
|
||||
#ifdef _WIN32
|
||||
/*
|
||||
* If VirtualAlloc can't allocate at the given address when one is
|
||||
* given, it fails and returns NULL.
|
||||
*/
|
||||
ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
|
||||
PAGE_READWRITE);
|
||||
#else
|
||||
/*
|
||||
* We don't use MAP_FIXED here, because it can cause the *replacement*
|
||||
* of existing mappings, and we only want to create new mappings.
|
||||
*/
|
||||
ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON,
|
||||
-1, 0);
|
||||
assert(ret != NULL);
|
||||
|
||||
if (ret == MAP_FAILED)
|
||||
ret = NULL;
|
||||
else if (addr != NULL && ret != addr) {
|
||||
/*
|
||||
* We succeeded in mapping memory, but not in the right place.
|
||||
*/
|
||||
if (munmap(ret, size) == -1) {
|
||||
char buf[BUFERROR_BUF];
|
||||
|
||||
buferror(get_errno(), buf, sizeof(buf));
|
||||
malloc_printf("<jemalloc: Error in munmap(): %s\n",
|
||||
buf);
|
||||
if (opt_abort)
|
||||
abort();
|
||||
}
|
||||
ret = NULL;
|
||||
}
|
||||
#endif
|
||||
assert(ret == NULL || (addr == NULL && ret != addr)
|
||||
|| (addr != NULL && ret == addr));
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static void
|
||||
pages_unmap(void *addr, size_t size)
|
||||
{
|
||||
|
||||
#ifdef _WIN32
|
||||
if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
|
||||
#else
|
||||
if (munmap(addr, size) == -1)
|
||||
#endif
|
||||
{
|
||||
char buf[BUFERROR_BUF];
|
||||
|
||||
buferror(get_errno(), buf, sizeof(buf));
|
||||
malloc_printf("<jemalloc>: Error in "
|
||||
#ifdef _WIN32
|
||||
"VirtualFree"
|
||||
#else
|
||||
"munmap"
|
||||
#endif
|
||||
"(): %s\n", buf);
|
||||
if (opt_abort)
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
static void *
|
||||
pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size)
|
||||
{
|
||||
void *ret = (void *)((uintptr_t)addr + leadsize);
|
||||
|
||||
assert(alloc_size >= leadsize + size);
|
||||
#ifdef _WIN32
|
||||
{
|
||||
void *new_addr;
|
||||
|
||||
pages_unmap(addr, alloc_size);
|
||||
new_addr = pages_map(ret, size);
|
||||
if (new_addr == ret)
|
||||
return (ret);
|
||||
if (new_addr)
|
||||
pages_unmap(new_addr, size);
|
||||
return (NULL);
|
||||
}
|
||||
#else
|
||||
{
|
||||
size_t trailsize = alloc_size - leadsize - size;
|
||||
|
||||
if (leadsize != 0)
|
||||
pages_unmap(addr, leadsize);
|
||||
if (trailsize != 0)
|
||||
pages_unmap((void *)((uintptr_t)ret + size), trailsize);
|
||||
return (ret);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
bool
|
||||
pages_purge(void *addr, size_t length)
|
||||
{
|
||||
bool unzeroed;
|
||||
|
||||
#ifdef _WIN32
|
||||
VirtualAlloc(addr, length, MEM_RESET, PAGE_READWRITE);
|
||||
unzeroed = true;
|
||||
#else
|
||||
# ifdef JEMALLOC_PURGE_MADVISE_DONTNEED
|
||||
# define JEMALLOC_MADV_PURGE MADV_DONTNEED
|
||||
# define JEMALLOC_MADV_ZEROS true
|
||||
# elif defined(JEMALLOC_PURGE_MADVISE_FREE)
|
||||
# define JEMALLOC_MADV_PURGE MADV_FREE
|
||||
# define JEMALLOC_MADV_ZEROS false
|
||||
# else
|
||||
# error "No method defined for purging unused dirty pages."
|
||||
# endif
|
||||
int err = madvise(addr, length, JEMALLOC_MADV_PURGE);
|
||||
unzeroed = (JEMALLOC_MADV_ZEROS == false || err != 0);
|
||||
# undef JEMALLOC_MADV_PURGE
|
||||
# undef JEMALLOC_MADV_ZEROS
|
||||
#endif
|
||||
return (unzeroed);
|
||||
}
|
||||
|
||||
static void *
|
||||
chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero)
|
||||
{
|
||||
void *ret, *pages;
|
||||
size_t alloc_size, leadsize;
|
||||
|
||||
alloc_size = size + alignment - PAGE;
|
||||
/* Beware size_t wrap-around. */
|
||||
if (alloc_size < size)
|
||||
return (NULL);
|
||||
do {
|
||||
void *pages;
|
||||
size_t leadsize;
|
||||
pages = pages_map(NULL, alloc_size, commit);
|
||||
pages = pages_map(NULL, alloc_size);
|
||||
if (pages == NULL)
|
||||
return (NULL);
|
||||
leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) -
|
||||
(uintptr_t)pages;
|
||||
ret = pages_trim(pages, alloc_size, leadsize, size, commit);
|
||||
ret = pages_trim(pages, alloc_size, leadsize, size);
|
||||
} while (ret == NULL);
|
||||
|
||||
assert(ret != NULL);
|
||||
@@ -30,8 +164,7 @@ chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero, bool *commit)
|
||||
}
|
||||
|
||||
void *
|
||||
chunk_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
bool *commit)
|
||||
chunk_alloc_mmap(size_t size, size_t alignment, bool *zero)
|
||||
{
|
||||
void *ret;
|
||||
size_t offset;
|
||||
@@ -52,14 +185,13 @@ chunk_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
assert(alignment != 0);
|
||||
assert((alignment & chunksize_mask) == 0);
|
||||
|
||||
ret = pages_map(new_addr, size, commit);
|
||||
if (ret == NULL || ret == new_addr)
|
||||
return (ret);
|
||||
assert(new_addr == NULL);
|
||||
ret = pages_map(NULL, size);
|
||||
if (ret == NULL)
|
||||
return (NULL);
|
||||
offset = ALIGNMENT_ADDR2OFFSET(ret, alignment);
|
||||
if (offset != 0) {
|
||||
pages_unmap(ret, size);
|
||||
return (chunk_alloc_mmap_slow(size, alignment, zero, commit));
|
||||
return (chunk_alloc_mmap_slow(size, alignment, zero));
|
||||
}
|
||||
|
||||
assert(ret != NULL);
|
||||
@@ -68,11 +200,11 @@ chunk_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_dalloc_mmap(void *chunk, size_t size)
|
||||
chunk_dealloc_mmap(void *chunk, size_t size)
|
||||
{
|
||||
|
||||
if (config_munmap)
|
||||
pages_unmap(chunk, size);
|
||||
|
||||
return (!config_munmap);
|
||||
return (config_munmap == false);
|
||||
}
|
||||
|
||||
82
src/ckh.c
82
src/ckh.c
@@ -40,8 +40,8 @@
|
||||
/******************************************************************************/
|
||||
/* Function prototypes for non-inline static functions. */
|
||||
|
||||
static bool ckh_grow(tsd_t *tsd, ckh_t *ckh);
|
||||
static void ckh_shrink(tsd_t *tsd, ckh_t *ckh);
|
||||
static bool ckh_grow(ckh_t *ckh);
|
||||
static void ckh_shrink(ckh_t *ckh);
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
@@ -99,8 +99,7 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key,
|
||||
* Cycle through the cells in the bucket, starting at a random position.
|
||||
* The randomness avoids worst-case search overhead as buckets fill up.
|
||||
*/
|
||||
offset = (unsigned)prng_lg_range_u64(&ckh->prng_state,
|
||||
LG_CKH_BUCKET_CELLS);
|
||||
prng32(offset, LG_CKH_BUCKET_CELLS, ckh->prng_state, CKH_A, CKH_C);
|
||||
for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) {
|
||||
cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) +
|
||||
((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))];
|
||||
@@ -142,8 +141,7 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
|
||||
* were an item for which both hashes indicated the same
|
||||
* bucket.
|
||||
*/
|
||||
i = (unsigned)prng_lg_range_u64(&ckh->prng_state,
|
||||
LG_CKH_BUCKET_CELLS);
|
||||
prng32(i, LG_CKH_BUCKET_CELLS, ckh->prng_state, CKH_A, CKH_C);
|
||||
cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i];
|
||||
assert(cell->key != NULL);
|
||||
|
||||
@@ -187,7 +185,7 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
|
||||
}
|
||||
|
||||
bucket = tbucket;
|
||||
if (!ckh_try_bucket_insert(ckh, bucket, key, data))
|
||||
if (ckh_try_bucket_insert(ckh, bucket, key, data) == false)
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
@@ -203,12 +201,12 @@ ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata)
|
||||
|
||||
/* Try to insert in primary bucket. */
|
||||
bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1);
|
||||
if (!ckh_try_bucket_insert(ckh, bucket, key, data))
|
||||
if (ckh_try_bucket_insert(ckh, bucket, key, data) == false)
|
||||
return (false);
|
||||
|
||||
/* Try to insert in secondary bucket. */
|
||||
bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1);
|
||||
if (!ckh_try_bucket_insert(ckh, bucket, key, data))
|
||||
if (ckh_try_bucket_insert(ckh, bucket, key, data) == false)
|
||||
return (false);
|
||||
|
||||
/*
|
||||
@@ -245,11 +243,12 @@ ckh_rebuild(ckh_t *ckh, ckhc_t *aTab)
|
||||
}
|
||||
|
||||
static bool
|
||||
ckh_grow(tsd_t *tsd, ckh_t *ckh)
|
||||
ckh_grow(ckh_t *ckh)
|
||||
{
|
||||
bool ret;
|
||||
ckhc_t *tab, *ttab;
|
||||
unsigned lg_prevbuckets, lg_curcells;
|
||||
size_t lg_curcells;
|
||||
unsigned lg_prevbuckets;
|
||||
|
||||
#ifdef CKH_COUNT
|
||||
ckh->ngrows++;
|
||||
@@ -267,12 +266,11 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh)
|
||||
|
||||
lg_curcells++;
|
||||
usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
|
||||
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) {
|
||||
if (usize == 0) {
|
||||
ret = true;
|
||||
goto label_return;
|
||||
}
|
||||
tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE,
|
||||
true, NULL, true, arena_ichoose(tsd, NULL));
|
||||
tab = (ckhc_t *)ipalloc(usize, CACHELINE, true);
|
||||
if (tab == NULL) {
|
||||
ret = true;
|
||||
goto label_return;
|
||||
@@ -283,13 +281,13 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh)
|
||||
tab = ttab;
|
||||
ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
|
||||
|
||||
if (!ckh_rebuild(ckh, tab)) {
|
||||
idalloctm(tsd_tsdn(tsd), tab, NULL, true, true);
|
||||
if (ckh_rebuild(ckh, tab) == false) {
|
||||
idalloc(tab);
|
||||
break;
|
||||
}
|
||||
|
||||
/* Rebuilding failed, so back out partially rebuilt table. */
|
||||
idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, true, true);
|
||||
idalloc(ckh->tab);
|
||||
ckh->tab = tab;
|
||||
ckh->lg_curbuckets = lg_prevbuckets;
|
||||
}
|
||||
@@ -300,11 +298,11 @@ label_return:
|
||||
}
|
||||
|
||||
static void
|
||||
ckh_shrink(tsd_t *tsd, ckh_t *ckh)
|
||||
ckh_shrink(ckh_t *ckh)
|
||||
{
|
||||
ckhc_t *tab, *ttab;
|
||||
size_t usize;
|
||||
unsigned lg_prevbuckets, lg_curcells;
|
||||
size_t lg_curcells, usize;
|
||||
unsigned lg_prevbuckets;
|
||||
|
||||
/*
|
||||
* It is possible (though unlikely, given well behaved hashes) that the
|
||||
@@ -313,10 +311,9 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh)
|
||||
lg_prevbuckets = ckh->lg_curbuckets;
|
||||
lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1;
|
||||
usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
|
||||
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
|
||||
if (usize == 0)
|
||||
return;
|
||||
tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, NULL,
|
||||
true, arena_ichoose(tsd, NULL));
|
||||
tab = (ckhc_t *)ipalloc(usize, CACHELINE, true);
|
||||
if (tab == NULL) {
|
||||
/*
|
||||
* An OOM error isn't worth propagating, since it doesn't
|
||||
@@ -330,8 +327,8 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh)
|
||||
tab = ttab;
|
||||
ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
|
||||
|
||||
if (!ckh_rebuild(ckh, tab)) {
|
||||
idalloctm(tsd_tsdn(tsd), tab, NULL, true, true);
|
||||
if (ckh_rebuild(ckh, tab) == false) {
|
||||
idalloc(tab);
|
||||
#ifdef CKH_COUNT
|
||||
ckh->nshrinks++;
|
||||
#endif
|
||||
@@ -339,7 +336,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh)
|
||||
}
|
||||
|
||||
/* Rebuilding failed, so back out partially rebuilt table. */
|
||||
idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, true, true);
|
||||
idalloc(ckh->tab);
|
||||
ckh->tab = tab;
|
||||
ckh->lg_curbuckets = lg_prevbuckets;
|
||||
#ifdef CKH_COUNT
|
||||
@@ -348,8 +345,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh)
|
||||
}
|
||||
|
||||
bool
|
||||
ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
|
||||
ckh_keycomp_t *keycomp)
|
||||
ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp)
|
||||
{
|
||||
bool ret;
|
||||
size_t mincells, usize;
|
||||
@@ -370,10 +366,10 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
|
||||
ckh->count = 0;
|
||||
|
||||
/*
|
||||
* Find the minimum power of 2 that is large enough to fit minitems
|
||||
* Find the minimum power of 2 that is large enough to fit aBaseCount
|
||||
* entries. We are using (2+,2) cuckoo hashing, which has an expected
|
||||
* maximum load factor of at least ~0.86, so 0.75 is a conservative load
|
||||
* factor that will typically allow mincells items to fit without ever
|
||||
* factor that will typically allow 2^aLgMinItems to fit without ever
|
||||
* growing the table.
|
||||
*/
|
||||
assert(LG_CKH_BUCKET_CELLS > 0);
|
||||
@@ -388,12 +384,11 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
|
||||
ckh->keycomp = keycomp;
|
||||
|
||||
usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE);
|
||||
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) {
|
||||
if (usize == 0) {
|
||||
ret = true;
|
||||
goto label_return;
|
||||
}
|
||||
ckh->tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true,
|
||||
NULL, true, arena_ichoose(tsd, NULL));
|
||||
ckh->tab = (ckhc_t *)ipalloc(usize, CACHELINE, true);
|
||||
if (ckh->tab == NULL) {
|
||||
ret = true;
|
||||
goto label_return;
|
||||
@@ -405,16 +400,16 @@ label_return:
|
||||
}
|
||||
|
||||
void
|
||||
ckh_delete(tsd_t *tsd, ckh_t *ckh)
|
||||
ckh_delete(ckh_t *ckh)
|
||||
{
|
||||
|
||||
assert(ckh != NULL);
|
||||
|
||||
#ifdef CKH_VERBOSE
|
||||
malloc_printf(
|
||||
"%s(%p): ngrows: %"FMTu64", nshrinks: %"FMTu64","
|
||||
" nshrinkfails: %"FMTu64", ninserts: %"FMTu64","
|
||||
" nrelocs: %"FMTu64"\n", __func__, ckh,
|
||||
"%s(%p): ngrows: %"PRIu64", nshrinks: %"PRIu64","
|
||||
" nshrinkfails: %"PRIu64", ninserts: %"PRIu64","
|
||||
" nrelocs: %"PRIu64"\n", __func__, ckh,
|
||||
(unsigned long long)ckh->ngrows,
|
||||
(unsigned long long)ckh->nshrinks,
|
||||
(unsigned long long)ckh->nshrinkfails,
|
||||
@@ -422,9 +417,9 @@ ckh_delete(tsd_t *tsd, ckh_t *ckh)
|
||||
(unsigned long long)ckh->nrelocs);
|
||||
#endif
|
||||
|
||||
idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, true, true);
|
||||
idalloc(ckh->tab);
|
||||
if (config_debug)
|
||||
memset(ckh, JEMALLOC_FREE_JUNK, sizeof(ckh_t));
|
||||
memset(ckh, 0x5a, sizeof(ckh_t));
|
||||
}
|
||||
|
||||
size_t
|
||||
@@ -457,7 +452,7 @@ ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data)
|
||||
}
|
||||
|
||||
bool
|
||||
ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data)
|
||||
ckh_insert(ckh_t *ckh, const void *key, const void *data)
|
||||
{
|
||||
bool ret;
|
||||
|
||||
@@ -469,7 +464,7 @@ ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data)
|
||||
#endif
|
||||
|
||||
while (ckh_try_insert(ckh, &key, &data)) {
|
||||
if (ckh_grow(tsd, ckh)) {
|
||||
if (ckh_grow(ckh)) {
|
||||
ret = true;
|
||||
goto label_return;
|
||||
}
|
||||
@@ -481,8 +476,7 @@ label_return:
|
||||
}
|
||||
|
||||
bool
|
||||
ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
|
||||
void **data)
|
||||
ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data)
|
||||
{
|
||||
size_t cell;
|
||||
|
||||
@@ -503,7 +497,7 @@ ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
|
||||
+ LG_CKH_BUCKET_CELLS - 2)) && ckh->lg_curbuckets
|
||||
> ckh->lg_minbuckets) {
|
||||
/* Ignore error due to OOM. */
|
||||
ckh_shrink(tsd, ckh);
|
||||
ckh_shrink(ckh);
|
||||
}
|
||||
|
||||
return (false);
|
||||
|
||||
80
src/extent.c
80
src/extent.c
@@ -3,75 +3,37 @@
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
/*
|
||||
* Round down to the nearest chunk size that can actually be requested during
|
||||
* normal huge allocation.
|
||||
*/
|
||||
JEMALLOC_INLINE_C size_t
|
||||
extent_quantize(size_t size)
|
||||
static inline int
|
||||
extent_szad_comp(extent_node_t *a, extent_node_t *b)
|
||||
{
|
||||
size_t ret;
|
||||
szind_t ind;
|
||||
int ret;
|
||||
size_t a_size = a->size;
|
||||
size_t b_size = b->size;
|
||||
|
||||
assert(size > 0);
|
||||
ret = (a_size > b_size) - (a_size < b_size);
|
||||
if (ret == 0) {
|
||||
uintptr_t a_addr = (uintptr_t)a->addr;
|
||||
uintptr_t b_addr = (uintptr_t)b->addr;
|
||||
|
||||
ind = size2index(size + 1);
|
||||
if (ind == 0) {
|
||||
/* Avoid underflow. */
|
||||
return (index2size(0));
|
||||
ret = (a_addr > b_addr) - (a_addr < b_addr);
|
||||
}
|
||||
ret = index2size(ind - 1);
|
||||
assert(ret <= size);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE_C int
|
||||
extent_sz_comp(const extent_node_t *a, const extent_node_t *b)
|
||||
/* Generate red-black tree functions. */
|
||||
rb_gen(, extent_tree_szad_, extent_tree_t, extent_node_t, link_szad,
|
||||
extent_szad_comp)
|
||||
|
||||
static inline int
|
||||
extent_ad_comp(extent_node_t *a, extent_node_t *b)
|
||||
{
|
||||
size_t a_qsize = extent_quantize(extent_node_size_get(a));
|
||||
size_t b_qsize = extent_quantize(extent_node_size_get(b));
|
||||
|
||||
return ((a_qsize > b_qsize) - (a_qsize < b_qsize));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE_C int
|
||||
extent_sn_comp(const extent_node_t *a, const extent_node_t *b)
|
||||
{
|
||||
size_t a_sn = extent_node_sn_get(a);
|
||||
size_t b_sn = extent_node_sn_get(b);
|
||||
|
||||
return ((a_sn > b_sn) - (a_sn < b_sn));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE_C int
|
||||
extent_ad_comp(const extent_node_t *a, const extent_node_t *b)
|
||||
{
|
||||
uintptr_t a_addr = (uintptr_t)extent_node_addr_get(a);
|
||||
uintptr_t b_addr = (uintptr_t)extent_node_addr_get(b);
|
||||
uintptr_t a_addr = (uintptr_t)a->addr;
|
||||
uintptr_t b_addr = (uintptr_t)b->addr;
|
||||
|
||||
return ((a_addr > b_addr) - (a_addr < b_addr));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE_C int
|
||||
extent_szsnad_comp(const extent_node_t *a, const extent_node_t *b)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = extent_sz_comp(a, b);
|
||||
if (ret != 0)
|
||||
return (ret);
|
||||
|
||||
ret = extent_sn_comp(a, b);
|
||||
if (ret != 0)
|
||||
return (ret);
|
||||
|
||||
ret = extent_ad_comp(a, b);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
/* Generate red-black tree functions. */
|
||||
rb_gen(, extent_tree_szsnad_, extent_tree_t, extent_node_t, szsnad_link,
|
||||
extent_szsnad_comp)
|
||||
|
||||
/* Generate red-black tree functions. */
|
||||
rb_gen(, extent_tree_ad_, extent_tree_t, extent_node_t, ad_link, extent_ad_comp)
|
||||
rb_gen(, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad,
|
||||
extent_ad_comp)
|
||||
|
||||
604
src/huge.c
604
src/huge.c
@@ -2,77 +2,44 @@
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
|
||||
/******************************************************************************/
|
||||
/* Data. */
|
||||
|
||||
static extent_node_t *
|
||||
huge_node_get(const void *ptr)
|
||||
{
|
||||
extent_node_t *node;
|
||||
uint64_t huge_nmalloc;
|
||||
uint64_t huge_ndalloc;
|
||||
size_t huge_allocated;
|
||||
|
||||
node = chunk_lookup(ptr, true);
|
||||
assert(!extent_node_achunk_get(node));
|
||||
malloc_mutex_t huge_mtx;
|
||||
|
||||
return (node);
|
||||
}
|
||||
/******************************************************************************/
|
||||
|
||||
static bool
|
||||
huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
|
||||
/* Tree of chunks that are stand-alone huge allocations. */
|
||||
static extent_tree_t huge;
|
||||
|
||||
void *
|
||||
huge_malloc(size_t size, bool zero, dss_prec_t dss_prec)
|
||||
{
|
||||
|
||||
assert(extent_node_addr_get(node) == ptr);
|
||||
assert(!extent_node_achunk_get(node));
|
||||
return (chunk_register(tsdn, ptr, node));
|
||||
}
|
||||
|
||||
static void
|
||||
huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
|
||||
{
|
||||
bool err;
|
||||
|
||||
err = huge_node_set(tsdn, ptr, node);
|
||||
assert(!err);
|
||||
}
|
||||
|
||||
static void
|
||||
huge_node_unset(const void *ptr, const extent_node_t *node)
|
||||
{
|
||||
|
||||
chunk_deregister(ptr, node);
|
||||
return (huge_palloc(size, chunksize, zero, dss_prec));
|
||||
}
|
||||
|
||||
void *
|
||||
huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
|
||||
{
|
||||
|
||||
assert(usize == s2u(usize));
|
||||
|
||||
return (huge_palloc(tsdn, arena, usize, chunksize, zero));
|
||||
}
|
||||
|
||||
void *
|
||||
huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
||||
bool zero)
|
||||
huge_palloc(size_t size, size_t alignment, bool zero, dss_prec_t dss_prec)
|
||||
{
|
||||
void *ret;
|
||||
size_t ausize;
|
||||
arena_t *iarena;
|
||||
size_t csize;
|
||||
extent_node_t *node;
|
||||
size_t sn;
|
||||
bool is_zeroed;
|
||||
|
||||
/* Allocate one or more contiguous chunks for this request. */
|
||||
|
||||
assert(!tsdn_null(tsdn) || arena != NULL);
|
||||
|
||||
ausize = sa2u(usize, alignment);
|
||||
if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS))
|
||||
csize = CHUNK_CEILING(size);
|
||||
if (csize == 0) {
|
||||
/* size is large enough to cause size_t wrap-around. */
|
||||
return (NULL);
|
||||
assert(ausize >= chunksize);
|
||||
}
|
||||
|
||||
/* Allocate an extent node with which to track the chunk. */
|
||||
iarena = (!tsdn_null(tsdn)) ? arena_ichoose(tsdn_tsd(tsdn), NULL) :
|
||||
a0get();
|
||||
node = ipallocztm(tsdn, CACHELINE_CEILING(sizeof(extent_node_t)),
|
||||
CACHELINE, false, NULL, true, iarena);
|
||||
node = base_node_alloc();
|
||||
if (node == NULL)
|
||||
return (NULL);
|
||||
|
||||
@@ -81,35 +48,145 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
||||
* it is possible to make correct junk/zero fill decisions below.
|
||||
*/
|
||||
is_zeroed = zero;
|
||||
if (likely(!tsdn_null(tsdn)))
|
||||
arena = arena_choose(tsdn_tsd(tsdn), arena);
|
||||
if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn,
|
||||
arena, usize, alignment, &sn, &is_zeroed)) == NULL) {
|
||||
idalloctm(tsdn, node, NULL, true, true);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
extent_node_init(node, arena, ret, usize, sn, is_zeroed, true);
|
||||
|
||||
if (huge_node_set(tsdn, ret, node)) {
|
||||
arena_chunk_dalloc_huge(tsdn, arena, ret, usize, sn);
|
||||
idalloctm(tsdn, node, NULL, true, true);
|
||||
ret = chunk_alloc(csize, alignment, false, &is_zeroed, dss_prec);
|
||||
if (ret == NULL) {
|
||||
base_node_dealloc(node);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
/* Insert node into huge. */
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
ql_elm_new(node, ql_link);
|
||||
ql_tail_insert(&arena->huge, node, ql_link);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
node->addr = ret;
|
||||
node->size = csize;
|
||||
|
||||
if (zero || (config_fill && unlikely(opt_zero))) {
|
||||
if (!is_zeroed)
|
||||
memset(ret, 0, usize);
|
||||
} else if (config_fill && unlikely(opt_junk_alloc))
|
||||
memset(ret, JEMALLOC_ALLOC_JUNK, usize);
|
||||
malloc_mutex_lock(&huge_mtx);
|
||||
extent_tree_ad_insert(&huge, node);
|
||||
if (config_stats) {
|
||||
stats_cactive_add(csize);
|
||||
huge_nmalloc++;
|
||||
huge_allocated += csize;
|
||||
}
|
||||
malloc_mutex_unlock(&huge_mtx);
|
||||
|
||||
arena_decay_tick(tsdn, arena);
|
||||
if (config_fill && zero == false) {
|
||||
if (opt_junk)
|
||||
memset(ret, 0xa5, csize);
|
||||
else if (opt_zero && is_zeroed == false)
|
||||
memset(ret, 0, csize);
|
||||
}
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
bool
|
||||
huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
|
||||
{
|
||||
|
||||
/*
|
||||
* Avoid moving the allocation if the size class can be left the same.
|
||||
*/
|
||||
if (oldsize > arena_maxclass
|
||||
&& CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size)
|
||||
&& CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) {
|
||||
assert(CHUNK_CEILING(oldsize) == oldsize);
|
||||
return (false);
|
||||
}
|
||||
|
||||
/* Reallocation would require a move. */
|
||||
return (true);
|
||||
}
|
||||
|
||||
void *
|
||||
huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
|
||||
size_t alignment, bool zero, bool try_tcache_dalloc, dss_prec_t dss_prec)
|
||||
{
|
||||
void *ret;
|
||||
size_t copysize;
|
||||
|
||||
/* Try to avoid moving the allocation. */
|
||||
if (huge_ralloc_no_move(ptr, oldsize, size, extra) == false)
|
||||
return (ptr);
|
||||
|
||||
/*
|
||||
* size and oldsize are different enough that we need to use a
|
||||
* different size class. In that case, fall back to allocating new
|
||||
* space and copying.
|
||||
*/
|
||||
if (alignment > chunksize)
|
||||
ret = huge_palloc(size + extra, alignment, zero, dss_prec);
|
||||
else
|
||||
ret = huge_malloc(size + extra, zero, dss_prec);
|
||||
|
||||
if (ret == NULL) {
|
||||
if (extra == 0)
|
||||
return (NULL);
|
||||
/* Try again, this time without extra. */
|
||||
if (alignment > chunksize)
|
||||
ret = huge_palloc(size, alignment, zero, dss_prec);
|
||||
else
|
||||
ret = huge_malloc(size, zero, dss_prec);
|
||||
|
||||
if (ret == NULL)
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
/*
|
||||
* Copy at most size bytes (not size+extra), since the caller has no
|
||||
* expectation that the extra bytes will be reliably preserved.
|
||||
*/
|
||||
copysize = (size < oldsize) ? size : oldsize;
|
||||
|
||||
#ifdef JEMALLOC_MREMAP
|
||||
/*
|
||||
* Use mremap(2) if this is a huge-->huge reallocation, and neither the
|
||||
* source nor the destination are in dss.
|
||||
*/
|
||||
if (oldsize >= chunksize && (config_dss == false || (chunk_in_dss(ptr)
|
||||
== false && chunk_in_dss(ret) == false))) {
|
||||
size_t newsize = huge_salloc(ret);
|
||||
|
||||
/*
|
||||
* Remove ptr from the tree of huge allocations before
|
||||
* performing the remap operation, in order to avoid the
|
||||
* possibility of another thread acquiring that mapping before
|
||||
* this one removes it from the tree.
|
||||
*/
|
||||
huge_dalloc(ptr, false);
|
||||
if (mremap(ptr, oldsize, newsize, MREMAP_MAYMOVE|MREMAP_FIXED,
|
||||
ret) == MAP_FAILED) {
|
||||
/*
|
||||
* Assuming no chunk management bugs in the allocator,
|
||||
* the only documented way an error can occur here is
|
||||
* if the application changed the map type for a
|
||||
* portion of the old allocation. This is firmly in
|
||||
* undefined behavior territory, so write a diagnostic
|
||||
* message, and optionally abort.
|
||||
*/
|
||||
char buf[BUFERROR_BUF];
|
||||
|
||||
buferror(get_errno(), buf, sizeof(buf));
|
||||
malloc_printf("<jemalloc>: Error in mremap(): %s\n",
|
||||
buf);
|
||||
if (opt_abort)
|
||||
abort();
|
||||
memcpy(ret, ptr, copysize);
|
||||
chunk_dealloc_mmap(ptr, oldsize);
|
||||
} else if (config_fill && zero == false && opt_junk && oldsize
|
||||
< newsize) {
|
||||
/*
|
||||
* mremap(2) clobbers the original mapping, so
|
||||
* junk/zero filling is not preserved. There is no
|
||||
* need to zero fill here, since any trailing
|
||||
* uninititialized memory is demand-zeroed by the
|
||||
* kernel, but junk filling must be redone.
|
||||
*/
|
||||
memset(ret + oldsize, 0xa5, newsize - oldsize);
|
||||
}
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
memcpy(ret, ptr, copysize);
|
||||
iqalloct(ptr, try_tcache_dalloc);
|
||||
}
|
||||
return (ret);
|
||||
}
|
||||
|
||||
@@ -121,13 +198,13 @@ static void
|
||||
huge_dalloc_junk(void *ptr, size_t usize)
|
||||
{
|
||||
|
||||
if (config_fill && have_dss && unlikely(opt_junk_free)) {
|
||||
if (config_fill && config_dss && opt_junk) {
|
||||
/*
|
||||
* Only bother junk filling if the chunk isn't about to be
|
||||
* unmapped.
|
||||
*/
|
||||
if (!config_munmap || (have_dss && chunk_in_dss(ptr)))
|
||||
memset(ptr, JEMALLOC_FREE_JUNK, usize);
|
||||
if (config_munmap == false || (config_dss && chunk_in_dss(ptr)))
|
||||
memset(ptr, 0x5a, usize);
|
||||
}
|
||||
}
|
||||
#ifdef JEMALLOC_JET
|
||||
@@ -136,342 +213,135 @@ huge_dalloc_junk(void *ptr, size_t usize)
|
||||
huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
|
||||
#endif
|
||||
|
||||
static void
|
||||
huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
||||
size_t usize_min, size_t usize_max, bool zero)
|
||||
void
|
||||
huge_dalloc(void *ptr, bool unmap)
|
||||
{
|
||||
size_t usize, usize_next;
|
||||
extent_node_t *node;
|
||||
arena_t *arena;
|
||||
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
|
||||
bool pre_zeroed, post_zeroed;
|
||||
extent_node_t *node, key;
|
||||
|
||||
/* Increase usize to incorporate extra. */
|
||||
for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
|
||||
<= oldsize; usize = usize_next)
|
||||
; /* Do nothing. */
|
||||
malloc_mutex_lock(&huge_mtx);
|
||||
|
||||
if (oldsize == usize)
|
||||
return;
|
||||
/* Extract from tree of huge allocations. */
|
||||
key.addr = ptr;
|
||||
node = extent_tree_ad_search(&huge, &key);
|
||||
assert(node != NULL);
|
||||
assert(node->addr == ptr);
|
||||
extent_tree_ad_remove(&huge, node);
|
||||
|
||||
node = huge_node_get(ptr);
|
||||
arena = extent_node_arena_get(node);
|
||||
pre_zeroed = extent_node_zeroed_get(node);
|
||||
|
||||
/* Fill if necessary (shrinking). */
|
||||
if (oldsize > usize) {
|
||||
size_t sdiff = oldsize - usize;
|
||||
if (config_fill && unlikely(opt_junk_free)) {
|
||||
memset((void *)((uintptr_t)ptr + usize),
|
||||
JEMALLOC_FREE_JUNK, sdiff);
|
||||
post_zeroed = false;
|
||||
} else {
|
||||
post_zeroed = !chunk_purge_wrapper(tsdn, arena,
|
||||
&chunk_hooks, ptr, CHUNK_CEILING(oldsize), usize,
|
||||
sdiff);
|
||||
}
|
||||
} else
|
||||
post_zeroed = pre_zeroed;
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
/* Update the size of the huge allocation. */
|
||||
huge_node_unset(ptr, node);
|
||||
assert(extent_node_size_get(node) != usize);
|
||||
extent_node_size_set(node, usize);
|
||||
huge_node_reset(tsdn, ptr, node);
|
||||
/* Update zeroed. */
|
||||
extent_node_zeroed_set(node, post_zeroed);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
|
||||
arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize);
|
||||
|
||||
/* Fill if necessary (growing). */
|
||||
if (oldsize < usize) {
|
||||
if (zero || (config_fill && unlikely(opt_zero))) {
|
||||
if (!pre_zeroed) {
|
||||
memset((void *)((uintptr_t)ptr + oldsize), 0,
|
||||
usize - oldsize);
|
||||
}
|
||||
} else if (config_fill && unlikely(opt_junk_alloc)) {
|
||||
memset((void *)((uintptr_t)ptr + oldsize),
|
||||
JEMALLOC_ALLOC_JUNK, usize - oldsize);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static bool
|
||||
huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
||||
size_t usize)
|
||||
{
|
||||
extent_node_t *node;
|
||||
arena_t *arena;
|
||||
chunk_hooks_t chunk_hooks;
|
||||
size_t cdiff;
|
||||
bool pre_zeroed, post_zeroed;
|
||||
|
||||
node = huge_node_get(ptr);
|
||||
arena = extent_node_arena_get(node);
|
||||
pre_zeroed = extent_node_zeroed_get(node);
|
||||
chunk_hooks = chunk_hooks_get(tsdn, arena);
|
||||
|
||||
assert(oldsize > usize);
|
||||
|
||||
/* Split excess chunks. */
|
||||
cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
|
||||
if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize),
|
||||
CHUNK_CEILING(usize), cdiff, true, arena->ind))
|
||||
return (true);
|
||||
|
||||
if (oldsize > usize) {
|
||||
size_t sdiff = oldsize - usize;
|
||||
if (config_fill && unlikely(opt_junk_free)) {
|
||||
huge_dalloc_junk((void *)((uintptr_t)ptr + usize),
|
||||
sdiff);
|
||||
post_zeroed = false;
|
||||
} else {
|
||||
post_zeroed = !chunk_purge_wrapper(tsdn, arena,
|
||||
&chunk_hooks, CHUNK_ADDR2BASE((uintptr_t)ptr +
|
||||
usize), CHUNK_CEILING(oldsize),
|
||||
CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff);
|
||||
}
|
||||
} else
|
||||
post_zeroed = pre_zeroed;
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
/* Update the size of the huge allocation. */
|
||||
huge_node_unset(ptr, node);
|
||||
extent_node_size_set(node, usize);
|
||||
huge_node_reset(tsdn, ptr, node);
|
||||
/* Update zeroed. */
|
||||
extent_node_zeroed_set(node, post_zeroed);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
|
||||
/* Zap the excess chunks. */
|
||||
arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize,
|
||||
extent_node_sn_get(node));
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
static bool
|
||||
huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
||||
size_t usize, bool zero) {
|
||||
extent_node_t *node;
|
||||
arena_t *arena;
|
||||
bool is_zeroed_subchunk, is_zeroed_chunk;
|
||||
|
||||
node = huge_node_get(ptr);
|
||||
arena = extent_node_arena_get(node);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
is_zeroed_subchunk = extent_node_zeroed_get(node);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
|
||||
/*
|
||||
* Use is_zeroed_chunk to detect whether the trailing memory is zeroed,
|
||||
* update extent's zeroed field, and zero as necessary.
|
||||
*/
|
||||
is_zeroed_chunk = false;
|
||||
if (arena_chunk_ralloc_huge_expand(tsdn, arena, ptr, oldsize, usize,
|
||||
&is_zeroed_chunk))
|
||||
return (true);
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
huge_node_unset(ptr, node);
|
||||
extent_node_size_set(node, usize);
|
||||
extent_node_zeroed_set(node, extent_node_zeroed_get(node) &&
|
||||
is_zeroed_chunk);
|
||||
huge_node_reset(tsdn, ptr, node);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
|
||||
if (zero || (config_fill && unlikely(opt_zero))) {
|
||||
if (!is_zeroed_subchunk) {
|
||||
memset((void *)((uintptr_t)ptr + oldsize), 0,
|
||||
CHUNK_CEILING(oldsize) - oldsize);
|
||||
}
|
||||
if (!is_zeroed_chunk) {
|
||||
memset((void *)((uintptr_t)ptr +
|
||||
CHUNK_CEILING(oldsize)), 0, usize -
|
||||
CHUNK_CEILING(oldsize));
|
||||
}
|
||||
} else if (config_fill && unlikely(opt_junk_alloc)) {
|
||||
memset((void *)((uintptr_t)ptr + oldsize), JEMALLOC_ALLOC_JUNK,
|
||||
usize - oldsize);
|
||||
if (config_stats) {
|
||||
stats_cactive_sub(node->size);
|
||||
huge_ndalloc++;
|
||||
huge_allocated -= node->size;
|
||||
}
|
||||
|
||||
return (false);
|
||||
malloc_mutex_unlock(&huge_mtx);
|
||||
|
||||
if (unmap)
|
||||
huge_dalloc_junk(node->addr, node->size);
|
||||
|
||||
chunk_dealloc(node->addr, node->size, unmap);
|
||||
|
||||
base_node_dealloc(node);
|
||||
}
|
||||
|
||||
bool
|
||||
huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
|
||||
size_t usize_max, bool zero)
|
||||
size_t
|
||||
huge_salloc(const void *ptr)
|
||||
{
|
||||
size_t ret;
|
||||
extent_node_t *node, key;
|
||||
|
||||
assert(s2u(oldsize) == oldsize);
|
||||
/* The following should have been caught by callers. */
|
||||
assert(usize_min > 0 && usize_max <= HUGE_MAXCLASS);
|
||||
malloc_mutex_lock(&huge_mtx);
|
||||
|
||||
/* Both allocations must be huge to avoid a move. */
|
||||
if (oldsize < chunksize || usize_max < chunksize)
|
||||
return (true);
|
||||
/* Extract from tree of huge allocations. */
|
||||
key.addr = __DECONST(void *, ptr);
|
||||
node = extent_tree_ad_search(&huge, &key);
|
||||
assert(node != NULL);
|
||||
|
||||
if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
|
||||
/* Attempt to expand the allocation in-place. */
|
||||
if (!huge_ralloc_no_move_expand(tsdn, ptr, oldsize, usize_max,
|
||||
zero)) {
|
||||
arena_decay_tick(tsdn, huge_aalloc(ptr));
|
||||
return (false);
|
||||
}
|
||||
/* Try again, this time with usize_min. */
|
||||
if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
|
||||
CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsdn,
|
||||
ptr, oldsize, usize_min, zero)) {
|
||||
arena_decay_tick(tsdn, huge_aalloc(ptr));
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
ret = node->size;
|
||||
|
||||
/*
|
||||
* Avoid moving the allocation if the existing chunk size accommodates
|
||||
* the new size.
|
||||
*/
|
||||
if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
|
||||
&& CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
|
||||
huge_ralloc_no_move_similar(tsdn, ptr, oldsize, usize_min,
|
||||
usize_max, zero);
|
||||
arena_decay_tick(tsdn, huge_aalloc(ptr));
|
||||
return (false);
|
||||
}
|
||||
malloc_mutex_unlock(&huge_mtx);
|
||||
|
||||
/* Attempt to shrink the allocation in-place. */
|
||||
if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
|
||||
if (!huge_ralloc_no_move_shrink(tsdn, ptr, oldsize,
|
||||
usize_max)) {
|
||||
arena_decay_tick(tsdn, huge_aalloc(ptr));
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
return (true);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static void *
|
||||
huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
||||
size_t alignment, bool zero)
|
||||
dss_prec_t
|
||||
huge_dss_prec_get(arena_t *arena)
|
||||
{
|
||||
|
||||
if (alignment <= chunksize)
|
||||
return (huge_malloc(tsdn, arena, usize, zero));
|
||||
return (huge_palloc(tsdn, arena, usize, alignment, zero));
|
||||
return (arena_dss_prec_get(choose_arena(arena)));
|
||||
}
|
||||
|
||||
void *
|
||||
huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
|
||||
size_t usize, size_t alignment, bool zero, tcache_t *tcache)
|
||||
prof_ctx_t *
|
||||
huge_prof_ctx_get(const void *ptr)
|
||||
{
|
||||
void *ret;
|
||||
size_t copysize;
|
||||
prof_ctx_t *ret;
|
||||
extent_node_t *node, key;
|
||||
|
||||
/* The following should have been caught by callers. */
|
||||
assert(usize > 0 && usize <= HUGE_MAXCLASS);
|
||||
malloc_mutex_lock(&huge_mtx);
|
||||
|
||||
/* Try to avoid moving the allocation. */
|
||||
if (!huge_ralloc_no_move(tsd_tsdn(tsd), ptr, oldsize, usize, usize,
|
||||
zero))
|
||||
return (ptr);
|
||||
/* Extract from tree of huge allocations. */
|
||||
key.addr = __DECONST(void *, ptr);
|
||||
node = extent_tree_ad_search(&huge, &key);
|
||||
assert(node != NULL);
|
||||
|
||||
/*
|
||||
* usize and oldsize are different enough that we need to use a
|
||||
* different size class. In that case, fall back to allocating new
|
||||
* space and copying.
|
||||
*/
|
||||
ret = huge_ralloc_move_helper(tsd_tsdn(tsd), arena, usize, alignment,
|
||||
zero);
|
||||
if (ret == NULL)
|
||||
return (NULL);
|
||||
ret = node->prof_ctx;
|
||||
|
||||
malloc_mutex_unlock(&huge_mtx);
|
||||
|
||||
copysize = (usize < oldsize) ? usize : oldsize;
|
||||
memcpy(ret, ptr, copysize);
|
||||
isqalloc(tsd, ptr, oldsize, tcache, true);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
void
|
||||
huge_dalloc(tsdn_t *tsdn, void *ptr)
|
||||
huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
|
||||
{
|
||||
extent_node_t *node;
|
||||
arena_t *arena;
|
||||
extent_node_t *node, key;
|
||||
|
||||
node = huge_node_get(ptr);
|
||||
arena = extent_node_arena_get(node);
|
||||
huge_node_unset(ptr, node);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
ql_remove(&arena->huge, node, ql_link);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
malloc_mutex_lock(&huge_mtx);
|
||||
|
||||
huge_dalloc_junk(extent_node_addr_get(node),
|
||||
extent_node_size_get(node));
|
||||
arena_chunk_dalloc_huge(tsdn, extent_node_arena_get(node),
|
||||
extent_node_addr_get(node), extent_node_size_get(node),
|
||||
extent_node_sn_get(node));
|
||||
idalloctm(tsdn, node, NULL, true, true);
|
||||
/* Extract from tree of huge allocations. */
|
||||
key.addr = __DECONST(void *, ptr);
|
||||
node = extent_tree_ad_search(&huge, &key);
|
||||
assert(node != NULL);
|
||||
|
||||
arena_decay_tick(tsdn, arena);
|
||||
node->prof_ctx = ctx;
|
||||
|
||||
malloc_mutex_unlock(&huge_mtx);
|
||||
}
|
||||
|
||||
arena_t *
|
||||
huge_aalloc(const void *ptr)
|
||||
bool
|
||||
huge_boot(void)
|
||||
{
|
||||
|
||||
return (extent_node_arena_get(huge_node_get(ptr)));
|
||||
}
|
||||
/* Initialize chunks data. */
|
||||
if (malloc_mutex_init(&huge_mtx))
|
||||
return (true);
|
||||
extent_tree_ad_new(&huge);
|
||||
|
||||
size_t
|
||||
huge_salloc(tsdn_t *tsdn, const void *ptr)
|
||||
{
|
||||
size_t size;
|
||||
extent_node_t *node;
|
||||
arena_t *arena;
|
||||
if (config_stats) {
|
||||
huge_nmalloc = 0;
|
||||
huge_ndalloc = 0;
|
||||
huge_allocated = 0;
|
||||
}
|
||||
|
||||
node = huge_node_get(ptr);
|
||||
arena = extent_node_arena_get(node);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
size = extent_node_size_get(node);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
|
||||
return (size);
|
||||
}
|
||||
|
||||
prof_tctx_t *
|
||||
huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
|
||||
{
|
||||
prof_tctx_t *tctx;
|
||||
extent_node_t *node;
|
||||
arena_t *arena;
|
||||
|
||||
node = huge_node_get(ptr);
|
||||
arena = extent_node_arena_get(node);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
tctx = extent_node_prof_tctx_get(node);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
|
||||
return (tctx);
|
||||
return (false);
|
||||
}
|
||||
|
||||
void
|
||||
huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx)
|
||||
huge_prefork(void)
|
||||
{
|
||||
extent_node_t *node;
|
||||
arena_t *arena;
|
||||
|
||||
node = huge_node_get(ptr);
|
||||
arena = extent_node_arena_get(node);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
extent_node_prof_tctx_set(node, tctx);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
malloc_mutex_prefork(&huge_mtx);
|
||||
}
|
||||
|
||||
void
|
||||
huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr)
|
||||
huge_postfork_parent(void)
|
||||
{
|
||||
|
||||
huge_prof_tctx_set(tsdn, ptr, (prof_tctx_t *)(uintptr_t)1U);
|
||||
malloc_mutex_postfork_parent(&huge_mtx);
|
||||
}
|
||||
|
||||
void
|
||||
huge_postfork_child(void)
|
||||
{
|
||||
|
||||
malloc_mutex_postfork_child(&huge_mtx);
|
||||
}
|
||||
|
||||
2929
src/jemalloc.c
2929
src/jemalloc.c
File diff suppressed because it is too large
Load Diff
33
src/mutex.c
33
src/mutex.c
@@ -69,19 +69,13 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
|
||||
#endif
|
||||
|
||||
bool
|
||||
malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank)
|
||||
malloc_mutex_init(malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
#ifdef _WIN32
|
||||
# if _WIN32_WINNT >= 0x0600
|
||||
InitializeSRWLock(&mutex->lock);
|
||||
# else
|
||||
if (!InitializeCriticalSectionAndSpinCount(&mutex->lock,
|
||||
_CRT_SPINCOUNT))
|
||||
return (true);
|
||||
# endif
|
||||
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
|
||||
mutex->lock = OS_UNFAIR_LOCK_INIT;
|
||||
#elif (defined(JEMALLOC_OSSPIN))
|
||||
mutex->lock = 0;
|
||||
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
|
||||
@@ -89,8 +83,8 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank)
|
||||
mutex->postponed_next = postponed_mutexes;
|
||||
postponed_mutexes = mutex;
|
||||
} else {
|
||||
if (_pthread_mutex_init_calloc_cb(&mutex->lock,
|
||||
bootstrap_calloc) != 0)
|
||||
if (_pthread_mutex_init_calloc_cb(&mutex->lock, base_calloc) !=
|
||||
0)
|
||||
return (true);
|
||||
}
|
||||
#else
|
||||
@@ -105,34 +99,31 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank)
|
||||
}
|
||||
pthread_mutexattr_destroy(&attr);
|
||||
#endif
|
||||
if (config_debug)
|
||||
witness_init(&mutex->witness, name, rank, NULL);
|
||||
return (false);
|
||||
}
|
||||
|
||||
void
|
||||
malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
malloc_mutex_prefork(malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
malloc_mutex_lock(tsdn, mutex);
|
||||
malloc_mutex_lock(mutex);
|
||||
}
|
||||
|
||||
void
|
||||
malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
malloc_mutex_postfork_parent(malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
malloc_mutex_unlock(tsdn, mutex);
|
||||
malloc_mutex_unlock(mutex);
|
||||
}
|
||||
|
||||
void
|
||||
malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
malloc_mutex_postfork_child(malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
#ifdef JEMALLOC_MUTEX_INIT_CB
|
||||
malloc_mutex_unlock(tsdn, mutex);
|
||||
malloc_mutex_unlock(mutex);
|
||||
#else
|
||||
if (malloc_mutex_init(mutex, mutex->witness.name,
|
||||
mutex->witness.rank)) {
|
||||
if (malloc_mutex_init(mutex)) {
|
||||
malloc_printf("<jemalloc>: Error re-initializing mutex in "
|
||||
"child\n");
|
||||
if (opt_abort)
|
||||
@@ -142,14 +133,14 @@ malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
}
|
||||
|
||||
bool
|
||||
malloc_mutex_boot(void)
|
||||
mutex_boot(void)
|
||||
{
|
||||
|
||||
#ifdef JEMALLOC_MUTEX_INIT_CB
|
||||
postpone_init = false;
|
||||
while (postponed_mutexes != NULL) {
|
||||
if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock,
|
||||
bootstrap_calloc) != 0)
|
||||
base_calloc) != 0)
|
||||
return (true);
|
||||
postponed_mutexes = postponed_mutexes->postponed_next;
|
||||
}
|
||||
|
||||
194
src/nstime.c
194
src/nstime.c
@@ -1,194 +0,0 @@
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
|
||||
#define BILLION UINT64_C(1000000000)
|
||||
|
||||
void
|
||||
nstime_init(nstime_t *time, uint64_t ns)
|
||||
{
|
||||
|
||||
time->ns = ns;
|
||||
}
|
||||
|
||||
void
|
||||
nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec)
|
||||
{
|
||||
|
||||
time->ns = sec * BILLION + nsec;
|
||||
}
|
||||
|
||||
uint64_t
|
||||
nstime_ns(const nstime_t *time)
|
||||
{
|
||||
|
||||
return (time->ns);
|
||||
}
|
||||
|
||||
uint64_t
|
||||
nstime_sec(const nstime_t *time)
|
||||
{
|
||||
|
||||
return (time->ns / BILLION);
|
||||
}
|
||||
|
||||
uint64_t
|
||||
nstime_nsec(const nstime_t *time)
|
||||
{
|
||||
|
||||
return (time->ns % BILLION);
|
||||
}
|
||||
|
||||
void
|
||||
nstime_copy(nstime_t *time, const nstime_t *source)
|
||||
{
|
||||
|
||||
*time = *source;
|
||||
}
|
||||
|
||||
int
|
||||
nstime_compare(const nstime_t *a, const nstime_t *b)
|
||||
{
|
||||
|
||||
return ((a->ns > b->ns) - (a->ns < b->ns));
|
||||
}
|
||||
|
||||
void
|
||||
nstime_add(nstime_t *time, const nstime_t *addend)
|
||||
{
|
||||
|
||||
assert(UINT64_MAX - time->ns >= addend->ns);
|
||||
|
||||
time->ns += addend->ns;
|
||||
}
|
||||
|
||||
void
|
||||
nstime_subtract(nstime_t *time, const nstime_t *subtrahend)
|
||||
{
|
||||
|
||||
assert(nstime_compare(time, subtrahend) >= 0);
|
||||
|
||||
time->ns -= subtrahend->ns;
|
||||
}
|
||||
|
||||
void
|
||||
nstime_imultiply(nstime_t *time, uint64_t multiplier)
|
||||
{
|
||||
|
||||
assert((((time->ns | multiplier) & (UINT64_MAX << (sizeof(uint64_t) <<
|
||||
2))) == 0) || ((time->ns * multiplier) / multiplier == time->ns));
|
||||
|
||||
time->ns *= multiplier;
|
||||
}
|
||||
|
||||
void
|
||||
nstime_idivide(nstime_t *time, uint64_t divisor)
|
||||
{
|
||||
|
||||
assert(divisor != 0);
|
||||
|
||||
time->ns /= divisor;
|
||||
}
|
||||
|
||||
uint64_t
|
||||
nstime_divide(const nstime_t *time, const nstime_t *divisor)
|
||||
{
|
||||
|
||||
assert(divisor->ns != 0);
|
||||
|
||||
return (time->ns / divisor->ns);
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
# define NSTIME_MONOTONIC true
|
||||
static void
|
||||
nstime_get(nstime_t *time)
|
||||
{
|
||||
FILETIME ft;
|
||||
uint64_t ticks_100ns;
|
||||
|
||||
GetSystemTimeAsFileTime(&ft);
|
||||
ticks_100ns = (((uint64_t)ft.dwHighDateTime) << 32) | ft.dwLowDateTime;
|
||||
|
||||
nstime_init(time, ticks_100ns * 100);
|
||||
}
|
||||
#elif JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
|
||||
# define NSTIME_MONOTONIC true
|
||||
static void
|
||||
nstime_get(nstime_t *time)
|
||||
{
|
||||
struct timespec ts;
|
||||
|
||||
clock_gettime(CLOCK_MONOTONIC_COARSE, &ts);
|
||||
nstime_init2(time, ts.tv_sec, ts.tv_nsec);
|
||||
}
|
||||
#elif JEMALLOC_HAVE_CLOCK_MONOTONIC
|
||||
# define NSTIME_MONOTONIC true
|
||||
static void
|
||||
nstime_get(nstime_t *time)
|
||||
{
|
||||
struct timespec ts;
|
||||
|
||||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
nstime_init2(time, ts.tv_sec, ts.tv_nsec);
|
||||
}
|
||||
#elif JEMALLOC_HAVE_MACH_ABSOLUTE_TIME
|
||||
# define NSTIME_MONOTONIC true
|
||||
static void
|
||||
nstime_get(nstime_t *time)
|
||||
{
|
||||
|
||||
nstime_init(time, mach_absolute_time());
|
||||
}
|
||||
#else
|
||||
# define NSTIME_MONOTONIC false
|
||||
static void
|
||||
nstime_get(nstime_t *time)
|
||||
{
|
||||
struct timeval tv;
|
||||
|
||||
gettimeofday(&tv, NULL);
|
||||
nstime_init2(time, tv.tv_sec, tv.tv_usec * 1000);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef JEMALLOC_JET
|
||||
#undef nstime_monotonic
|
||||
#define nstime_monotonic JEMALLOC_N(n_nstime_monotonic)
|
||||
#endif
|
||||
bool
|
||||
nstime_monotonic(void)
|
||||
{
|
||||
|
||||
return (NSTIME_MONOTONIC);
|
||||
#undef NSTIME_MONOTONIC
|
||||
}
|
||||
#ifdef JEMALLOC_JET
|
||||
#undef nstime_monotonic
|
||||
#define nstime_monotonic JEMALLOC_N(nstime_monotonic)
|
||||
nstime_monotonic_t *nstime_monotonic = JEMALLOC_N(n_nstime_monotonic);
|
||||
#endif
|
||||
|
||||
#ifdef JEMALLOC_JET
|
||||
#undef nstime_update
|
||||
#define nstime_update JEMALLOC_N(n_nstime_update)
|
||||
#endif
|
||||
bool
|
||||
nstime_update(nstime_t *time)
|
||||
{
|
||||
nstime_t old_time;
|
||||
|
||||
nstime_copy(&old_time, time);
|
||||
nstime_get(time);
|
||||
|
||||
/* Handle non-monotonic clocks. */
|
||||
if (unlikely(nstime_compare(&old_time, time) > 0)) {
|
||||
nstime_copy(time, &old_time);
|
||||
return (true);
|
||||
}
|
||||
|
||||
return (false);
|
||||
}
|
||||
#ifdef JEMALLOC_JET
|
||||
#undef nstime_update
|
||||
#define nstime_update JEMALLOC_N(nstime_update)
|
||||
nstime_update_t *nstime_update = JEMALLOC_N(n_nstime_update);
|
||||
#endif
|
||||
302
src/pages.c
302
src/pages.c
@@ -1,302 +0,0 @@
|
||||
#define JEMALLOC_PAGES_C_
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
|
||||
#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
|
||||
#include <sys/sysctl.h>
|
||||
#endif
|
||||
|
||||
/******************************************************************************/
|
||||
/* Data. */
|
||||
|
||||
#ifndef _WIN32
|
||||
# define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE)
|
||||
# define PAGES_PROT_DECOMMIT (PROT_NONE)
|
||||
static int mmap_flags;
|
||||
#endif
|
||||
static bool os_overcommits;
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
void *
|
||||
pages_map(void *addr, size_t size, bool *commit)
|
||||
{
|
||||
void *ret;
|
||||
|
||||
assert(size != 0);
|
||||
|
||||
if (os_overcommits)
|
||||
*commit = true;
|
||||
|
||||
#ifdef _WIN32
|
||||
/*
|
||||
* If VirtualAlloc can't allocate at the given address when one is
|
||||
* given, it fails and returns NULL.
|
||||
*/
|
||||
ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0),
|
||||
PAGE_READWRITE);
|
||||
#else
|
||||
/*
|
||||
* We don't use MAP_FIXED here, because it can cause the *replacement*
|
||||
* of existing mappings, and we only want to create new mappings.
|
||||
*/
|
||||
{
|
||||
int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
|
||||
|
||||
ret = mmap(addr, size, prot, mmap_flags, -1, 0);
|
||||
}
|
||||
assert(ret != NULL);
|
||||
|
||||
if (ret == MAP_FAILED)
|
||||
ret = NULL;
|
||||
else if (addr != NULL && ret != addr) {
|
||||
/*
|
||||
* We succeeded in mapping memory, but not in the right place.
|
||||
*/
|
||||
pages_unmap(ret, size);
|
||||
ret = NULL;
|
||||
}
|
||||
#endif
|
||||
assert(ret == NULL || (addr == NULL && ret != addr)
|
||||
|| (addr != NULL && ret == addr));
|
||||
return (ret);
|
||||
}
|
||||
|
||||
void
|
||||
pages_unmap(void *addr, size_t size)
|
||||
{
|
||||
|
||||
#ifdef _WIN32
|
||||
if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
|
||||
#else
|
||||
if (munmap(addr, size) == -1)
|
||||
#endif
|
||||
{
|
||||
char buf[BUFERROR_BUF];
|
||||
|
||||
buferror(get_errno(), buf, sizeof(buf));
|
||||
malloc_printf("<jemalloc>: Error in "
|
||||
#ifdef _WIN32
|
||||
"VirtualFree"
|
||||
#else
|
||||
"munmap"
|
||||
#endif
|
||||
"(): %s\n", buf);
|
||||
if (opt_abort)
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
void *
|
||||
pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,
|
||||
bool *commit)
|
||||
{
|
||||
void *ret = (void *)((uintptr_t)addr + leadsize);
|
||||
|
||||
assert(alloc_size >= leadsize + size);
|
||||
#ifdef _WIN32
|
||||
{
|
||||
void *new_addr;
|
||||
|
||||
pages_unmap(addr, alloc_size);
|
||||
new_addr = pages_map(ret, size, commit);
|
||||
if (new_addr == ret)
|
||||
return (ret);
|
||||
if (new_addr)
|
||||
pages_unmap(new_addr, size);
|
||||
return (NULL);
|
||||
}
|
||||
#else
|
||||
{
|
||||
size_t trailsize = alloc_size - leadsize - size;
|
||||
|
||||
if (leadsize != 0)
|
||||
pages_unmap(addr, leadsize);
|
||||
if (trailsize != 0)
|
||||
pages_unmap((void *)((uintptr_t)ret + size), trailsize);
|
||||
return (ret);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
static bool
|
||||
pages_commit_impl(void *addr, size_t size, bool commit)
|
||||
{
|
||||
|
||||
if (os_overcommits)
|
||||
return (true);
|
||||
|
||||
#ifdef _WIN32
|
||||
return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT,
|
||||
PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT)));
|
||||
#else
|
||||
{
|
||||
int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
|
||||
void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED,
|
||||
-1, 0);
|
||||
if (result == MAP_FAILED)
|
||||
return (true);
|
||||
if (result != addr) {
|
||||
/*
|
||||
* We succeeded in mapping memory, but not in the right
|
||||
* place.
|
||||
*/
|
||||
pages_unmap(result, size);
|
||||
return (true);
|
||||
}
|
||||
return (false);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
bool
|
||||
pages_commit(void *addr, size_t size)
|
||||
{
|
||||
|
||||
return (pages_commit_impl(addr, size, true));
|
||||
}
|
||||
|
||||
bool
|
||||
pages_decommit(void *addr, size_t size)
|
||||
{
|
||||
|
||||
return (pages_commit_impl(addr, size, false));
|
||||
}
|
||||
|
||||
bool
|
||||
pages_purge(void *addr, size_t size)
|
||||
{
|
||||
bool unzeroed;
|
||||
|
||||
#ifdef _WIN32
|
||||
VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
|
||||
unzeroed = true;
|
||||
#elif (defined(JEMALLOC_PURGE_MADVISE_FREE) || \
|
||||
defined(JEMALLOC_PURGE_MADVISE_DONTNEED))
|
||||
# if defined(JEMALLOC_PURGE_MADVISE_FREE)
|
||||
# define JEMALLOC_MADV_PURGE MADV_FREE
|
||||
# define JEMALLOC_MADV_ZEROS false
|
||||
# elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
|
||||
# define JEMALLOC_MADV_PURGE MADV_DONTNEED
|
||||
# define JEMALLOC_MADV_ZEROS true
|
||||
# else
|
||||
# error No madvise(2) flag defined for purging unused dirty pages
|
||||
# endif
|
||||
int err = madvise(addr, size, JEMALLOC_MADV_PURGE);
|
||||
unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0);
|
||||
# undef JEMALLOC_MADV_PURGE
|
||||
# undef JEMALLOC_MADV_ZEROS
|
||||
#else
|
||||
/* Last resort no-op. */
|
||||
unzeroed = true;
|
||||
#endif
|
||||
return (unzeroed);
|
||||
}
|
||||
|
||||
bool
|
||||
pages_huge(void *addr, size_t size)
|
||||
{
|
||||
|
||||
assert(PAGE_ADDR2BASE(addr) == addr);
|
||||
assert(PAGE_CEILING(size) == size);
|
||||
|
||||
#ifdef JEMALLOC_THP
|
||||
return (madvise(addr, size, MADV_HUGEPAGE) != 0);
|
||||
#else
|
||||
return (false);
|
||||
#endif
|
||||
}
|
||||
|
||||
bool
|
||||
pages_nohuge(void *addr, size_t size)
|
||||
{
|
||||
|
||||
assert(PAGE_ADDR2BASE(addr) == addr);
|
||||
assert(PAGE_CEILING(size) == size);
|
||||
|
||||
#ifdef JEMALLOC_THP
|
||||
return (madvise(addr, size, MADV_NOHUGEPAGE) != 0);
|
||||
#else
|
||||
return (false);
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
|
||||
static bool
|
||||
os_overcommits_sysctl(void)
|
||||
{
|
||||
int vm_overcommit;
|
||||
size_t sz;
|
||||
|
||||
sz = sizeof(vm_overcommit);
|
||||
if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0)
|
||||
return (false); /* Error. */
|
||||
|
||||
return ((vm_overcommit & 0x3) == 0);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
|
||||
/*
|
||||
* Use syscall(2) rather than {open,read,close}(2) when possible to avoid
|
||||
* reentry during bootstrapping if another library has interposed system call
|
||||
* wrappers.
|
||||
*/
|
||||
static bool
|
||||
os_overcommits_proc(void)
|
||||
{
|
||||
int fd;
|
||||
char buf[1];
|
||||
ssize_t nread;
|
||||
|
||||
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
|
||||
fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY);
|
||||
#else
|
||||
fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
|
||||
#endif
|
||||
if (fd == -1)
|
||||
return (false); /* Error. */
|
||||
|
||||
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read)
|
||||
nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf));
|
||||
#else
|
||||
nread = read(fd, &buf, sizeof(buf));
|
||||
#endif
|
||||
|
||||
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
|
||||
syscall(SYS_close, fd);
|
||||
#else
|
||||
close(fd);
|
||||
#endif
|
||||
|
||||
if (nread < 1)
|
||||
return (false); /* Error. */
|
||||
/*
|
||||
* /proc/sys/vm/overcommit_memory meanings:
|
||||
* 0: Heuristic overcommit.
|
||||
* 1: Always overcommit.
|
||||
* 2: Never overcommit.
|
||||
*/
|
||||
return (buf[0] == '0' || buf[0] == '1');
|
||||
}
|
||||
#endif
|
||||
|
||||
void
|
||||
pages_boot(void)
|
||||
{
|
||||
|
||||
#ifndef _WIN32
|
||||
mmap_flags = MAP_PRIVATE | MAP_ANON;
|
||||
#endif
|
||||
|
||||
#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
|
||||
os_overcommits = os_overcommits_sysctl();
|
||||
#elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY)
|
||||
os_overcommits = os_overcommits_proc();
|
||||
# ifdef MAP_NORESERVE
|
||||
if (os_overcommits)
|
||||
mmap_flags |= MAP_NORESERVE;
|
||||
# endif
|
||||
#else
|
||||
os_overcommits = false;
|
||||
#endif
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
#define JEMALLOC_PRNG_C_
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
2051
src/prof.c
2051
src/prof.c
File diff suppressed because it is too large
Load Diff
138
src/quarantine.c
138
src/quarantine.c
@@ -2,33 +2,34 @@
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
|
||||
/*
|
||||
* Quarantine pointers close to NULL are used to encode state information that
|
||||
* quarantine pointers close to NULL are used to encode state information that
|
||||
* is used for cleaning up during thread shutdown.
|
||||
*/
|
||||
#define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1)
|
||||
#define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2)
|
||||
#define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY
|
||||
|
||||
/******************************************************************************/
|
||||
/* Data. */
|
||||
|
||||
malloc_tsd_data(, quarantine, quarantine_t *, NULL)
|
||||
|
||||
/******************************************************************************/
|
||||
/* Function prototypes for non-inline static functions. */
|
||||
|
||||
static quarantine_t *quarantine_grow(tsd_t *tsd, quarantine_t *quarantine);
|
||||
static void quarantine_drain_one(tsdn_t *tsdn, quarantine_t *quarantine);
|
||||
static void quarantine_drain(tsdn_t *tsdn, quarantine_t *quarantine,
|
||||
size_t upper_bound);
|
||||
static quarantine_t *quarantine_grow(quarantine_t *quarantine);
|
||||
static void quarantine_drain_one(quarantine_t *quarantine);
|
||||
static void quarantine_drain(quarantine_t *quarantine, size_t upper_bound);
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
static quarantine_t *
|
||||
quarantine_init(tsdn_t *tsdn, size_t lg_maxobjs)
|
||||
quarantine_t *
|
||||
quarantine_init(size_t lg_maxobjs)
|
||||
{
|
||||
quarantine_t *quarantine;
|
||||
size_t size;
|
||||
|
||||
size = offsetof(quarantine_t, objs) + ((ZU(1) << lg_maxobjs) *
|
||||
sizeof(quarantine_obj_t));
|
||||
quarantine = (quarantine_t *)iallocztm(tsdn, size, size2index(size),
|
||||
false, NULL, true, arena_get(TSDN_NULL, 0, true), true);
|
||||
quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) +
|
||||
((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)));
|
||||
if (quarantine == NULL)
|
||||
return (NULL);
|
||||
quarantine->curbytes = 0;
|
||||
@@ -36,36 +37,19 @@ quarantine_init(tsdn_t *tsdn, size_t lg_maxobjs)
|
||||
quarantine->first = 0;
|
||||
quarantine->lg_maxobjs = lg_maxobjs;
|
||||
|
||||
quarantine_tsd_set(&quarantine);
|
||||
|
||||
return (quarantine);
|
||||
}
|
||||
|
||||
void
|
||||
quarantine_alloc_hook_work(tsd_t *tsd)
|
||||
{
|
||||
quarantine_t *quarantine;
|
||||
|
||||
if (!tsd_nominal(tsd))
|
||||
return;
|
||||
|
||||
quarantine = quarantine_init(tsd_tsdn(tsd), LG_MAXOBJS_INIT);
|
||||
/*
|
||||
* Check again whether quarantine has been initialized, because
|
||||
* quarantine_init() may have triggered recursive initialization.
|
||||
*/
|
||||
if (tsd_quarantine_get(tsd) == NULL)
|
||||
tsd_quarantine_set(tsd, quarantine);
|
||||
else
|
||||
idalloctm(tsd_tsdn(tsd), quarantine, NULL, true, true);
|
||||
}
|
||||
|
||||
static quarantine_t *
|
||||
quarantine_grow(tsd_t *tsd, quarantine_t *quarantine)
|
||||
quarantine_grow(quarantine_t *quarantine)
|
||||
{
|
||||
quarantine_t *ret;
|
||||
|
||||
ret = quarantine_init(tsd_tsdn(tsd), quarantine->lg_maxobjs + 1);
|
||||
ret = quarantine_init(quarantine->lg_maxobjs + 1);
|
||||
if (ret == NULL) {
|
||||
quarantine_drain_one(tsd_tsdn(tsd), quarantine);
|
||||
quarantine_drain_one(quarantine);
|
||||
return (quarantine);
|
||||
}
|
||||
|
||||
@@ -87,18 +71,17 @@ quarantine_grow(tsd_t *tsd, quarantine_t *quarantine)
|
||||
memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
|
||||
sizeof(quarantine_obj_t));
|
||||
}
|
||||
idalloctm(tsd_tsdn(tsd), quarantine, NULL, true, true);
|
||||
idalloc(quarantine);
|
||||
|
||||
tsd_quarantine_set(tsd, ret);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static void
|
||||
quarantine_drain_one(tsdn_t *tsdn, quarantine_t *quarantine)
|
||||
quarantine_drain_one(quarantine_t *quarantine)
|
||||
{
|
||||
quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
|
||||
assert(obj->usize == isalloc(tsdn, obj->ptr, config_prof));
|
||||
idalloctm(tsdn, obj->ptr, NULL, false, true);
|
||||
assert(obj->usize == isalloc(obj->ptr, config_prof));
|
||||
idalloc(obj->ptr);
|
||||
quarantine->curbytes -= obj->usize;
|
||||
quarantine->curobjs--;
|
||||
quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
|
||||
@@ -106,24 +89,33 @@ quarantine_drain_one(tsdn_t *tsdn, quarantine_t *quarantine)
|
||||
}
|
||||
|
||||
static void
|
||||
quarantine_drain(tsdn_t *tsdn, quarantine_t *quarantine, size_t upper_bound)
|
||||
quarantine_drain(quarantine_t *quarantine, size_t upper_bound)
|
||||
{
|
||||
|
||||
while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
|
||||
quarantine_drain_one(tsdn, quarantine);
|
||||
quarantine_drain_one(quarantine);
|
||||
}
|
||||
|
||||
void
|
||||
quarantine(tsd_t *tsd, void *ptr)
|
||||
quarantine(void *ptr)
|
||||
{
|
||||
quarantine_t *quarantine;
|
||||
size_t usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
|
||||
size_t usize = isalloc(ptr, config_prof);
|
||||
|
||||
cassert(config_fill);
|
||||
assert(opt_quarantine);
|
||||
|
||||
if ((quarantine = tsd_quarantine_get(tsd)) == NULL) {
|
||||
idalloctm(tsd_tsdn(tsd), ptr, NULL, false, true);
|
||||
quarantine = *quarantine_tsd_get();
|
||||
if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) {
|
||||
if (quarantine == QUARANTINE_STATE_PURGATORY) {
|
||||
/*
|
||||
* Make a note that quarantine() was called after
|
||||
* quarantine_cleanup() was called.
|
||||
*/
|
||||
quarantine = QUARANTINE_STATE_REINCARNATED;
|
||||
quarantine_tsd_set(&quarantine);
|
||||
}
|
||||
idalloc(ptr);
|
||||
return;
|
||||
}
|
||||
/*
|
||||
@@ -133,11 +125,11 @@ quarantine(tsd_t *tsd, void *ptr)
|
||||
if (quarantine->curbytes + usize > opt_quarantine) {
|
||||
size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
|
||||
- usize : 0;
|
||||
quarantine_drain(tsd_tsdn(tsd), quarantine, upper_bound);
|
||||
quarantine_drain(quarantine, upper_bound);
|
||||
}
|
||||
/* Grow the quarantine ring buffer if it's full. */
|
||||
if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
|
||||
quarantine = quarantine_grow(tsd, quarantine);
|
||||
quarantine = quarantine_grow(quarantine);
|
||||
/* quarantine_grow() must free a slot if it fails to grow. */
|
||||
assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
|
||||
/* Append ptr if its size doesn't exceed the quarantine size. */
|
||||
@@ -149,35 +141,59 @@ quarantine(tsd_t *tsd, void *ptr)
|
||||
obj->usize = usize;
|
||||
quarantine->curbytes += usize;
|
||||
quarantine->curobjs++;
|
||||
if (config_fill && unlikely(opt_junk_free)) {
|
||||
if (config_fill && opt_junk) {
|
||||
/*
|
||||
* Only do redzone validation if Valgrind isn't in
|
||||
* operation.
|
||||
*/
|
||||
if ((!config_valgrind || likely(!in_valgrind))
|
||||
if ((config_valgrind == false || opt_valgrind == false)
|
||||
&& usize <= SMALL_MAXCLASS)
|
||||
arena_quarantine_junk_small(ptr, usize);
|
||||
else
|
||||
memset(ptr, JEMALLOC_FREE_JUNK, usize);
|
||||
memset(ptr, 0x5a, usize);
|
||||
}
|
||||
} else {
|
||||
assert(quarantine->curbytes == 0);
|
||||
idalloctm(tsd_tsdn(tsd), ptr, NULL, false, true);
|
||||
idalloc(ptr);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
quarantine_cleanup(tsd_t *tsd)
|
||||
quarantine_cleanup(void *arg)
|
||||
{
|
||||
quarantine_t *quarantine;
|
||||
quarantine_t *quarantine = *(quarantine_t **)arg;
|
||||
|
||||
if (!config_fill)
|
||||
return;
|
||||
|
||||
quarantine = tsd_quarantine_get(tsd);
|
||||
if (quarantine != NULL) {
|
||||
quarantine_drain(tsd_tsdn(tsd), quarantine, 0);
|
||||
idalloctm(tsd_tsdn(tsd), quarantine, NULL, true, true);
|
||||
tsd_quarantine_set(tsd, NULL);
|
||||
if (quarantine == QUARANTINE_STATE_REINCARNATED) {
|
||||
/*
|
||||
* Another destructor deallocated memory after this destructor
|
||||
* was called. Reset quarantine to QUARANTINE_STATE_PURGATORY
|
||||
* in order to receive another callback.
|
||||
*/
|
||||
quarantine = QUARANTINE_STATE_PURGATORY;
|
||||
quarantine_tsd_set(&quarantine);
|
||||
} else if (quarantine == QUARANTINE_STATE_PURGATORY) {
|
||||
/*
|
||||
* The previous time this destructor was called, we set the key
|
||||
* to QUARANTINE_STATE_PURGATORY so that other destructors
|
||||
* wouldn't cause re-creation of the quarantine. This time, do
|
||||
* nothing, so that the destructor will not be called again.
|
||||
*/
|
||||
} else if (quarantine != NULL) {
|
||||
quarantine_drain(quarantine, 0);
|
||||
idalloc(quarantine);
|
||||
quarantine = QUARANTINE_STATE_PURGATORY;
|
||||
quarantine_tsd_set(&quarantine);
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
quarantine_boot(void)
|
||||
{
|
||||
|
||||
cassert(config_fill);
|
||||
|
||||
if (quarantine_tsd_boot())
|
||||
return (true);
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
151
src/rtree.c
151
src/rtree.c
@@ -1,76 +1,73 @@
|
||||
#define JEMALLOC_RTREE_C_
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
|
||||
static unsigned
|
||||
hmin(unsigned ha, unsigned hb)
|
||||
rtree_t *
|
||||
rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc)
|
||||
{
|
||||
rtree_t *ret;
|
||||
unsigned bits_per_level, bits_in_leaf, height, i;
|
||||
|
||||
return (ha < hb ? ha : hb);
|
||||
}
|
||||
|
||||
/* Only the most significant bits of keys passed to rtree_[gs]et() are used. */
|
||||
bool
|
||||
rtree_new(rtree_t *rtree, unsigned bits, rtree_node_alloc_t *alloc,
|
||||
rtree_node_dalloc_t *dalloc)
|
||||
{
|
||||
unsigned bits_in_leaf, height, i;
|
||||
|
||||
assert(RTREE_HEIGHT_MAX == ((ZU(1) << (LG_SIZEOF_PTR+3)) /
|
||||
RTREE_BITS_PER_LEVEL));
|
||||
assert(bits > 0 && bits <= (sizeof(uintptr_t) << 3));
|
||||
|
||||
bits_in_leaf = (bits % RTREE_BITS_PER_LEVEL) == 0 ? RTREE_BITS_PER_LEVEL
|
||||
: (bits % RTREE_BITS_PER_LEVEL);
|
||||
bits_per_level = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void *)))) - 1;
|
||||
bits_in_leaf = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(uint8_t)))) - 1;
|
||||
if (bits > bits_in_leaf) {
|
||||
height = 1 + (bits - bits_in_leaf) / RTREE_BITS_PER_LEVEL;
|
||||
if ((height-1) * RTREE_BITS_PER_LEVEL + bits_in_leaf != bits)
|
||||
height = 1 + (bits - bits_in_leaf) / bits_per_level;
|
||||
if ((height-1) * bits_per_level + bits_in_leaf != bits)
|
||||
height++;
|
||||
} else
|
||||
} else {
|
||||
height = 1;
|
||||
assert((height-1) * RTREE_BITS_PER_LEVEL + bits_in_leaf == bits);
|
||||
|
||||
rtree->alloc = alloc;
|
||||
rtree->dalloc = dalloc;
|
||||
rtree->height = height;
|
||||
|
||||
/* Root level. */
|
||||
rtree->levels[0].subtree = NULL;
|
||||
rtree->levels[0].bits = (height > 1) ? RTREE_BITS_PER_LEVEL :
|
||||
bits_in_leaf;
|
||||
rtree->levels[0].cumbits = rtree->levels[0].bits;
|
||||
/* Interior levels. */
|
||||
for (i = 1; i < height-1; i++) {
|
||||
rtree->levels[i].subtree = NULL;
|
||||
rtree->levels[i].bits = RTREE_BITS_PER_LEVEL;
|
||||
rtree->levels[i].cumbits = rtree->levels[i-1].cumbits +
|
||||
RTREE_BITS_PER_LEVEL;
|
||||
}
|
||||
/* Leaf level. */
|
||||
assert((height-1) * bits_per_level + bits_in_leaf >= bits);
|
||||
|
||||
ret = (rtree_t*)alloc(offsetof(rtree_t, level2bits) +
|
||||
(sizeof(unsigned) * height));
|
||||
if (ret == NULL)
|
||||
return (NULL);
|
||||
memset(ret, 0, offsetof(rtree_t, level2bits) + (sizeof(unsigned) *
|
||||
height));
|
||||
|
||||
ret->alloc = alloc;
|
||||
ret->dalloc = dalloc;
|
||||
if (malloc_mutex_init(&ret->mutex)) {
|
||||
if (dalloc != NULL)
|
||||
dalloc(ret);
|
||||
return (NULL);
|
||||
}
|
||||
ret->height = height;
|
||||
if (height > 1) {
|
||||
rtree->levels[height-1].subtree = NULL;
|
||||
rtree->levels[height-1].bits = bits_in_leaf;
|
||||
rtree->levels[height-1].cumbits = bits;
|
||||
}
|
||||
if ((height-1) * bits_per_level + bits_in_leaf > bits) {
|
||||
ret->level2bits[0] = (bits - bits_in_leaf) %
|
||||
bits_per_level;
|
||||
} else
|
||||
ret->level2bits[0] = bits_per_level;
|
||||
for (i = 1; i < height-1; i++)
|
||||
ret->level2bits[i] = bits_per_level;
|
||||
ret->level2bits[height-1] = bits_in_leaf;
|
||||
} else
|
||||
ret->level2bits[0] = bits;
|
||||
|
||||
/* Compute lookup table to be used by rtree_start_level(). */
|
||||
for (i = 0; i < RTREE_HEIGHT_MAX; i++) {
|
||||
rtree->start_level[i] = hmin(RTREE_HEIGHT_MAX - 1 - i, height -
|
||||
1);
|
||||
ret->root = (void**)alloc(sizeof(void *) << ret->level2bits[0]);
|
||||
if (ret->root == NULL) {
|
||||
if (dalloc != NULL)
|
||||
dalloc(ret);
|
||||
return (NULL);
|
||||
}
|
||||
memset(ret->root, 0, sizeof(void *) << ret->level2bits[0]);
|
||||
|
||||
return (false);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static void
|
||||
rtree_delete_subtree(rtree_t *rtree, rtree_node_elm_t *node, unsigned level)
|
||||
rtree_delete_subtree(rtree_t *rtree, void **node, unsigned level)
|
||||
{
|
||||
|
||||
if (level + 1 < rtree->height) {
|
||||
if (level < rtree->height - 1) {
|
||||
size_t nchildren, i;
|
||||
|
||||
nchildren = ZU(1) << rtree->levels[level].bits;
|
||||
nchildren = ZU(1) << rtree->level2bits[level];
|
||||
for (i = 0; i < nchildren; i++) {
|
||||
rtree_node_elm_t *child = node[i].child;
|
||||
void **child = (void **)node[i];
|
||||
if (child != NULL)
|
||||
rtree_delete_subtree(rtree, child, level + 1);
|
||||
}
|
||||
@@ -81,52 +78,28 @@ rtree_delete_subtree(rtree_t *rtree, rtree_node_elm_t *node, unsigned level)
|
||||
void
|
||||
rtree_delete(rtree_t *rtree)
|
||||
{
|
||||
unsigned i;
|
||||
|
||||
for (i = 0; i < rtree->height; i++) {
|
||||
rtree_node_elm_t *subtree = rtree->levels[i].subtree;
|
||||
if (subtree != NULL)
|
||||
rtree_delete_subtree(rtree, subtree, i);
|
||||
}
|
||||
rtree_delete_subtree(rtree, rtree->root, 0);
|
||||
rtree->dalloc(rtree);
|
||||
}
|
||||
|
||||
static rtree_node_elm_t *
|
||||
rtree_node_init(rtree_t *rtree, unsigned level, rtree_node_elm_t **elmp)
|
||||
{
|
||||
rtree_node_elm_t *node;
|
||||
|
||||
if (atomic_cas_p((void **)elmp, NULL, RTREE_NODE_INITIALIZING)) {
|
||||
spin_t spinner;
|
||||
|
||||
/*
|
||||
* Another thread is already in the process of initializing.
|
||||
* Spin-wait until initialization is complete.
|
||||
*/
|
||||
spin_init(&spinner);
|
||||
do {
|
||||
spin_adaptive(&spinner);
|
||||
node = atomic_read_p((void **)elmp);
|
||||
} while (node == RTREE_NODE_INITIALIZING);
|
||||
} else {
|
||||
node = rtree->alloc(ZU(1) << rtree->levels[level].bits);
|
||||
if (node == NULL)
|
||||
return (NULL);
|
||||
atomic_write_p((void **)elmp, node);
|
||||
}
|
||||
|
||||
return (node);
|
||||
}
|
||||
|
||||
rtree_node_elm_t *
|
||||
rtree_subtree_read_hard(rtree_t *rtree, unsigned level)
|
||||
void
|
||||
rtree_prefork(rtree_t *rtree)
|
||||
{
|
||||
|
||||
return (rtree_node_init(rtree, level, &rtree->levels[level].subtree));
|
||||
malloc_mutex_prefork(&rtree->mutex);
|
||||
}
|
||||
|
||||
rtree_node_elm_t *
|
||||
rtree_child_read_hard(rtree_t *rtree, rtree_node_elm_t *elm, unsigned level)
|
||||
void
|
||||
rtree_postfork_parent(rtree_t *rtree)
|
||||
{
|
||||
|
||||
return (rtree_node_init(rtree, level+1, &elm->child));
|
||||
malloc_mutex_postfork_parent(&rtree->mutex);
|
||||
}
|
||||
|
||||
void
|
||||
rtree_postfork_child(rtree_t *rtree)
|
||||
{
|
||||
|
||||
malloc_mutex_postfork_child(&rtree->mutex);
|
||||
}
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
#define JEMALLOC_SPIN_C_
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
1401
src/stats.c
Executable file → Normal file
1401
src/stats.c
Executable file → Normal file
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user