lihao/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-22 04:03:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: lihao:4e903a0a32ba278085f5050ee13442b8e4d2e844
Branches Tags
lihao:dev lihao:master lihao:stable-4 lihao:stable-3
lihao:5.3.1 lihao:5.3.0 lihao:5.2.1 lihao:5.2.0 lihao:5.1.0 lihao:5.0.1 lihao:5.0.0 lihao:4.5.0 lihao:4.4.0 lihao:4.3.1 lihao:4.3.0 lihao:4.2.1 lihao:4.2.0 lihao:4.1.1 lihao:4.1.0 lihao:4.0.4 lihao:4.0.3 lihao:4.0.2 lihao:4.0.1 lihao:4.0.0 lihao:3.6.0 lihao:3.5.1 lihao:3.5.0 lihao:3.4.1 lihao:3.4.0 lihao:3.3.1 lihao:3.3.0 lihao:3.2.0 lihao:3.1.0 lihao:3.0.0 lihao:2.2.5 lihao:2.2.4 lihao:2.2.3 lihao:2.2.2 lihao:2.2.1 lihao:2.2.0 lihao:2.1.3 lihao:2.1.2 lihao:2.1.1 lihao:2.1.0 lihao:2.0.1 lihao:2.0.0 lihao:1.0.3 lihao:1.0.2 lihao:1.0.1 lihao:1.0.0 lihao:0.0.0
..
compare: lihao:master
Branches Tags
lihao:dev lihao:master lihao:stable-4 lihao:stable-3
lihao:5.3.1 lihao:5.3.0 lihao:5.2.1 lihao:5.2.0 lihao:5.1.0 lihao:5.0.1 lihao:5.0.0 lihao:4.5.0 lihao:4.4.0 lihao:4.3.1 lihao:4.3.0 lihao:4.2.1 lihao:4.2.0 lihao:4.1.1 lihao:4.1.0 lihao:4.0.4 lihao:4.0.3 lihao:4.0.2 lihao:4.0.1 lihao:4.0.0 lihao:3.6.0 lihao:3.5.1 lihao:3.5.0 lihao:3.4.1 lihao:3.4.0 lihao:3.3.1 lihao:3.3.0 lihao:3.2.0 lihao:3.1.0 lihao:3.0.0 lihao:2.2.5 lihao:2.2.4 lihao:2.2.3 lihao:2.2.2 lihao:2.2.1 lihao:2.2.0 lihao:2.1.3 lihao:2.1.2 lihao:2.1.1 lihao:2.1.0 lihao:2.0.1 lihao:2.0.0 lihao:1.0.3 lihao:1.0.2 lihao:1.0.1 lihao:1.0.0 lihao:0.0.0

1 Commits
4e903a0a32 ... master

Author SHA1 Message Date
guangli-dai
3a02c75e04 Merge branch 'dev' 2026-04-24 22:10:20 -07:00
224 changed files with 8244 additions and 11246 deletions
Expand all files Collapse all files

5
.appveyor.yml
View File

@@ -1,5 +1,3 @@
image: Visual Studio 2022
version: '{build}'
environment:
@@ -39,8 +37,7 @@ environment:
install:
- set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH%
- if "%MSVC%"=="amd64" call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
- if "%MSVC%"=="x86" call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars32.bat"
- if defined MSVC call "c:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %MSVC%
- if defined MSVC pacman --noconfirm -Rsc mingw-w64-%CPU%-gcc gcc
- pacman --noconfirm -Syuu
- pacman --noconfirm -S autoconf

2
.github/workflows/check_formatting.yaml vendored
View File

@@ -5,6 +5,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out repository
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Check for trailing whitespace
run: scripts/check_trailing_whitespace.sh

2
.github/workflows/freebsd-ci.yml vendored
View File

@@ -25,7 +25,7 @@ jobs:
name: FreeBSD (${{ matrix.arch }}, debug=${{ matrix.debug }}, prof=${{ matrix.prof }}${{ matrix.uncommon && ', uncommon' || '' }})
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
with:
fetch-depth: 1

19
.github/workflows/linux-ci.yml vendored
View File

@@ -508,19 +508,9 @@ jobs:
CXX: g++
CONFIGURE_FLAGS: "--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof"
EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
- env:
CC: gcc
CXX: g++
CONFIGURE_FLAGS: --enable-cxx-infallible-new
EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
- env:
CC: gcc
CXX: g++
CONFIGURE_FLAGS: "--enable-cxx-infallible-new --enable-debug"
EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Show OS version
run: |
@@ -647,14 +637,9 @@ jobs:
CXX: g++
CONFIGURE_FLAGS: "--with-malloc-conf=background_thread:true"
EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
- env:
CC: gcc
CXX: g++
CONFIGURE_FLAGS: --enable-cxx-infallible-new
EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Show OS version
run: |

14
.github/workflows/macos-ci.yml vendored
View File

@@ -60,14 +60,9 @@ jobs:
CXX: g++
CONFIGURE_FLAGS: "--with-malloc-conf=percpu_arena:percpu"
EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
- env:
CC: gcc
CXX: g++
CONFIGURE_FLAGS: --enable-cxx-infallible-new
EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Show OS version
run: |
@@ -167,14 +162,9 @@ jobs:
CXX: g++
CONFIGURE_FLAGS: "--with-malloc-conf=percpu_arena:percpu"
EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
- env:
CC: gcc
CXX: g++
CONFIGURE_FLAGS: --enable-cxx-infallible-new
EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Show OS version
run: |

6
.github/workflows/static_analysis.yaml vendored
View File

@@ -7,7 +7,7 @@ jobs:
# We build libunwind ourselves because sadly the version
# provided by Ubuntu via apt-get is much too old.
- name: Check out libunwind
uses: actions/checkout@v6
uses: actions/checkout@v4
with:
repository: libunwind/libunwind
path: libunwind
@@ -23,7 +23,7 @@ jobs:
cd ..
rm -rf libunwind
- name: Check out repository
uses: actions/checkout@v6
uses: actions/checkout@v4
# We download LLVM directly from the latest stable release
# on GitHub, because this tends to be much newer than the
# version available via apt-get in Ubuntu.
@@ -54,7 +54,7 @@ jobs:
scripts/run_static_analysis.sh static_analysis_results "$GITHUB_OUTPUT"
- name: Upload static analysis results
if: ${{ steps.run_static_analysis.outputs.HAS_STATIC_ANALYSIS_RESULTS }} == '1'
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
with:
name: static_analysis_results
path: static_analysis_results

7
.github/workflows/windows-ci.yml vendored
View File

@@ -52,14 +52,9 @@ jobs:
CXX: cl.exe
CROSS_COMPILE_32BIT: yes
CONFIGURE_FLAGS: --enable-debug
- env:
CC: gcc
CXX: g++
CONFIGURE_FLAGS: --enable-cxx-infallible-new
EXTRA_CFLAGS: -fcommon
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Show OS version
shell: cmd

20
INSTALL.md
View File

@@ -217,15 +217,6 @@ any of the following arguments (not a definitive list) to 'configure':
Disable C++ integration. This will cause new and delete operator
implementations to be omitted.
* `--enable-cxx-infallible-new`
Make the throwing `operator new` abort on allocation failure (logging the
requested size) instead of throwing `std::bad_alloc`; the `std::nothrow`
overloads still return null. Disabled by default, and has no effect when
C++ integration is disabled (`--disable-cxx`). When enabled, under LTO
this lets the compiler treat `operator new` as non-throwing and elide
exception-handling cleanup in callers.
* `--with-xslroot=<path>`
Specify where to find DocBook XSL stylesheets when building the
@@ -420,7 +411,7 @@ differ in their ease of use and flexibility.
### With MSVC solutions
This is the easy, but less flexible approach. It doesn't let you specify
arguments to the `configure` script.
1. Install Cygwin with at least the following packages:
* autoconf
* autogen
@@ -428,19 +419,18 @@ arguments to the `configure` script.
* grep
* sed
2. Install Visual Studio with Visual C++. We currently test on 2022 and provide
solution files back to 2015.
2. Install Visual Studio 2015 or 2017 with Visual C++
3. Add Cygwin\bin to the PATH environment variable
4. Open "x64 Native Tools Command Prompt" for your version of Visual Studio
4. Open "x64 Native Tools Command Prompt for VS 2017"
(note: x86/x64 doesn't matter at this point)
5. Generate header files:
sh -c "CC=cl ./autogen.sh"
6. Now the project can be opened and built in Visual Studio using the
corresponding solution files in the `msvc\` folder.
6. Now the project can be opened and built in Visual Studio:
msvc\jemalloc_vc2017.sln
### With MSYS
This is a more involved approach that offers the same configuration flexibility

99
Makefile.in
View File

@@ -95,10 +95,7 @@ LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix)
BINS := $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh $(objroot)bin/jeprof
C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h
C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/jemalloc_fork.c \
$(srcroot)src/jemalloc_init.c \
$(srcroot)src/arena.c \
$(srcroot)src/arenas_management.c \
$(srcroot)src/background_thread.c \
$(srcroot)src/base.c \
$(srcroot)src/bin.c \
@@ -124,6 +121,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/fxp.c \
$(srcroot)src/san.c \
$(srcroot)src/san_bump.c \
$(srcroot)src/hook.c \
$(srcroot)src/hpa.c \
$(srcroot)src/hpa_central.c \
$(srcroot)src/hpa_hooks.c \
@@ -132,7 +130,6 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/inspect.c \
$(srcroot)src/large.c \
$(srcroot)src/log.c \
$(srcroot)src/malloc_dispatch.c \
$(srcroot)src/malloc_io.c \
$(srcroot)src/conf.c \
$(srcroot)src/mutex.c \
@@ -206,12 +203,12 @@ TESTS_UNIT := \
$(srcroot)test/unit/a0.c \
$(srcroot)test/unit/arena_decay.c \
$(srcroot)test/unit/arena_reset.c \
$(srcroot)test/unit/arenas_management.c \
$(srcroot)test/unit/atomic.c \
$(srcroot)test/unit/background_thread.c \
$(srcroot)test/unit/background_thread_enable.c \
$(srcroot)test/unit/background_thread_init.c \
$(srcroot)test/unit/base.c \
$(srcroot)test/unit/batch_alloc.c \
$(srcroot)test/unit/bin.c \
$(srcroot)test/unit/binshard.c \
$(srcroot)test/unit/bitmap.c \
@@ -229,10 +226,7 @@ TESTS_UNIT := \
$(srcroot)test/unit/div.c \
$(srcroot)test/unit/double_free.c \
$(srcroot)test/unit/edata_cache.c \
$(srcroot)test/unit/emap.c \
$(srcroot)test/unit/emitter.c \
$(srcroot)test/unit/eset.c \
$(srcroot)test/unit/extent_dss.c \
$(srcroot)test/unit/extent_quantize.c \
${srcroot}test/unit/fb.c \
$(srcroot)test/unit/fork.c \
@@ -240,19 +234,16 @@ TESTS_UNIT := \
${srcroot}test/unit/san.c \
${srcroot}test/unit/san_bump.c \
$(srcroot)test/unit/hash.c \
$(srcroot)test/unit/hook.c \
$(srcroot)test/unit/hpa.c \
$(srcroot)test/unit/hpa_central.c \
$(srcroot)test/unit/hpa_sec_integration.c \
$(srcroot)test/unit/hpa_thp_always.c \
$(srcroot)test/unit/hpa_vectorized_madvise.c \
$(srcroot)test/unit/hpa_vectorized_madvise_large_batch.c \
$(srcroot)test/unit/hpa_background_thread.c \
$(srcroot)test/unit/hpa_pageslab_packing.c \
$(srcroot)test/unit/hpdata.c \
$(srcroot)test/unit/extent_alloc_flags.c \
$(srcroot)test/unit/huge.c \
$(srcroot)test/unit/inspect.c \
$(srcroot)test/unit/jemalloc_init.c \
$(srcroot)test/unit/junk.c \
$(srcroot)test/unit/junk_alloc.c \
$(srcroot)test/unit/junk_free.c \
@@ -261,7 +252,6 @@ TESTS_UNIT := \
$(srcroot)test/unit/log.c \
$(srcroot)test/unit/mallctl.c \
$(srcroot)test/unit/malloc_conf_2.c \
$(srcroot)test/unit/malloc_dispatch.c \
$(srcroot)test/unit/malloc_io.c \
$(srcroot)test/unit/math.c \
$(srcroot)test/unit/mpsc_queue.c \
@@ -271,7 +261,6 @@ TESTS_UNIT := \
$(srcroot)test/unit/ncached_max.c \
$(srcroot)test/unit/oversize_threshold.c \
$(srcroot)test/unit/pa.c \
$(srcroot)test/unit/pac.c \
$(srcroot)test/unit/pack.c \
$(srcroot)test/unit/pages.c \
$(srcroot)test/unit/peak.c \
@@ -300,6 +289,7 @@ TESTS_UNIT := \
$(srcroot)test/unit/safety_check.c \
$(srcroot)test/unit/sc.c \
$(srcroot)test/unit/sec.c \
$(srcroot)test/unit/seq.c \
$(srcroot)test/unit/SFMT.c \
$(srcroot)test/unit/size_check.c \
$(srcroot)test/unit/size_classes.c \
@@ -311,7 +301,6 @@ TESTS_UNIT := \
$(srcroot)test/unit/sz.c \
$(srcroot)test/unit/tcache_init.c \
$(srcroot)test/unit/tcache_max.c \
$(srcroot)test/unit/tcache_gc.c \
$(srcroot)test/unit/test_hooks.c \
$(srcroot)test/unit/thread_event.c \
$(srcroot)test/unit/ticker.c \
@@ -325,7 +314,8 @@ TESTS_UNIT := \
$(srcroot)test/unit/zero_reallocs.c
ifeq (@enable_prof@, 1)
TESTS_UNIT += \
$(srcroot)test/unit/arena_reset_prof.c
$(srcroot)test/unit/arena_reset_prof.c \
$(srcroot)test/unit/batch_alloc_prof.c
endif
TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
$(srcroot)test/integration/allocated.c \
@@ -347,14 +337,9 @@ TESTS_INTEGRATION += \
endif
ifeq (@enable_cxx@, 1)
CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp
TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp
ifeq (@enable_cxx_exceptions@, 1)
ifeq (@enable_cxx_infallible_new@, 1)
TESTS_INTEGRATION_CPP += $(srcroot)test/integration/cpp/infallible_new.cpp
else
TESTS_INTEGRATION_CPP += $(srcroot)test/integration/cpp/failing_new.cpp
endif
endif
TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp \
$(srcroot)test/integration/cpp/infallible_new_true.cpp \
$(srcroot)test/integration/cpp/infallible_new_false.cpp
else
CPP_SRCS :=
TESTS_INTEGRATION_CPP :=
@@ -362,7 +347,9 @@ endif
TESTS_ANALYZE := $(srcroot)test/analyze/prof_bias.c \
$(srcroot)test/analyze/rand.c \
$(srcroot)test/analyze/sizes.c
TESTS_STRESS := $(srcroot)test/stress/fill_flush.c \
TESTS_STRESS := $(srcroot)test/stress/batch_alloc.c \
$(srcroot)test/stress/fill_flush.c \
$(srcroot)test/stress/hookbench.c \
$(srcroot)test/stress/large_microbench.c \
$(srcroot)test/stress/mallctl.c \
$(srcroot)test/stress/microbench.c
@@ -622,38 +609,38 @@ build_lib: build_lib_static
endif
install_bin:
$(INSTALL) -d '$(BINDIR)'
$(INSTALL) -d $(BINDIR)
@for b in $(BINS); do \
echo "$(INSTALL) -m 755 $$b '$(BINDIR)'"; \
$(INSTALL) -m 755 $$b '$(BINDIR)'; \
echo "$(INSTALL) -m 755 $$b $(BINDIR)"; \
$(INSTALL) -m 755 $$b $(BINDIR); \
done
install_include:
$(INSTALL) -d '$(INCLUDEDIR)/jemalloc'
$(INSTALL) -d $(INCLUDEDIR)/jemalloc
@for h in $(C_HDRS); do \
echo "$(INSTALL) -m 644 $$h '$(INCLUDEDIR)/jemalloc'"; \
$(INSTALL) -m 644 $$h '$(INCLUDEDIR)/jemalloc'; \
echo "$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc"; \
$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc; \
done
install_lib_shared: $(DSOS)
$(INSTALL) -d '$(LIBDIR)'
$(INSTALL) -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) '$(LIBDIR)'
$(INSTALL) -d $(LIBDIR)
$(INSTALL) -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR)
ifneq ($(SOREV),$(SO))
ln -sf $(LIBJEMALLOC).$(SOREV) '$(LIBDIR)/$(LIBJEMALLOC).$(SO)'
ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO)
endif
install_lib_static: $(STATIC_LIBS)
$(INSTALL) -d '$(LIBDIR)'
$(INSTALL) -d $(LIBDIR)
@for l in $(STATIC_LIBS); do \
echo "$(INSTALL) -m 644 $$l '$(LIBDIR)'"; \
$(INSTALL) -m 644 $$l '$(LIBDIR)'; \
echo "$(INSTALL) -m 755 $$l $(LIBDIR)"; \
$(INSTALL) -m 755 $$l $(LIBDIR); \
done
install_lib_pc: $(PC)
$(INSTALL) -d '$(LIBDIR)/pkgconfig'
$(INSTALL) -d $(LIBDIR)/pkgconfig
@for l in $(PC); do \
echo "$(INSTALL) -m 644 $$l '$(LIBDIR)/pkgconfig'"; \
$(INSTALL) -m 644 $$l '$(LIBDIR)/pkgconfig'; \
echo "$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig"; \
$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig; \
done
ifeq ($(enable_shared), 1)
@@ -665,17 +652,17 @@ endif
install_lib: install_lib_pc
install_doc_html: build_doc_html
$(INSTALL) -d '$(DATADIR)/doc/jemalloc$(install_suffix)'
$(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix)
@for d in $(DOCS_HTML); do \
echo "$(INSTALL) -m 644 $$d '$(DATADIR)/doc/jemalloc$(install_suffix)'"; \
$(INSTALL) -m 644 $$d '$(DATADIR)/doc/jemalloc$(install_suffix)'; \
echo "$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \
$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \
done
install_doc_man: build_doc_man
$(INSTALL) -d '$(MANDIR)/man3'
$(INSTALL) -d $(MANDIR)/man3
@for d in $(DOCS_MAN3); do \
echo "$(INSTALL) -m 644 $$d '$(MANDIR)/man3'"; \
$(INSTALL) -m 644 $$d '$(MANDIR)/man3'; \
echo "$(INSTALL) -m 644 $$d $(MANDIR)/man3"; \
$(INSTALL) -m 644 $$d $(MANDIR)/man3; \
done
install_doc: install_doc_html install_doc_man
@@ -687,23 +674,23 @@ install: install_doc
endif
uninstall_bin:
$(RM) -v $(foreach b,$(notdir $(BINS)),'$(BINDIR)/$(b)')
$(RM) -v $(foreach b,$(notdir $(BINS)),$(BINDIR)/$(b))
uninstall_include:
$(RM) -v $(foreach h,$(notdir $(C_HDRS)),'$(INCLUDEDIR)/jemalloc/$(h)')
rmdir -v '$(INCLUDEDIR)/jemalloc'
$(RM) -v $(foreach h,$(notdir $(C_HDRS)),$(INCLUDEDIR)/jemalloc/$(h))
rmdir -v $(INCLUDEDIR)/jemalloc
uninstall_lib_shared:
$(RM) -v '$(LIBDIR)/$(LIBJEMALLOC).$(SOREV)'
$(RM) -v $(LIBDIR)/$(LIBJEMALLOC).$(SOREV)
ifneq ($(SOREV),$(SO))
$(RM) -v '$(LIBDIR)/$(LIBJEMALLOC).$(SO)'
$(RM) -v $(LIBDIR)/$(LIBJEMALLOC).$(SO)
endif
uninstall_lib_static:
$(RM) -v $(foreach l,$(notdir $(STATIC_LIBS)),'$(LIBDIR)/$(l)')
$(RM) -v $(foreach l,$(notdir $(STATIC_LIBS)),$(LIBDIR)/$(l))
uninstall_lib_pc:
$(RM) -v $(foreach p,$(notdir $(PC)),'$(LIBDIR)/pkgconfig/$(p)')
$(RM) -v $(foreach p,$(notdir $(PC)),$(LIBDIR)/pkgconfig/$(p))
ifeq ($(enable_shared), 1)
uninstall_lib: uninstall_lib_shared
@@ -714,11 +701,11 @@ endif
uninstall_lib: uninstall_lib_pc
uninstall_doc_html:
$(RM) -v $(foreach d,$(notdir $(DOCS_HTML)),'$(DATADIR)/doc/jemalloc$(install_suffix)/$(d)')
rmdir -v '$(DATADIR)/doc/jemalloc$(install_suffix)'
$(RM) -v $(foreach d,$(notdir $(DOCS_HTML)),$(DATADIR)/doc/jemalloc$(install_suffix)/$(d))
rmdir -v $(DATADIR)/doc/jemalloc$(install_suffix)
uninstall_doc_man:
$(RM) -v $(foreach d,$(notdir $(DOCS_MAN3)),'$(MANDIR)/man3/$(d)')
$(RM) -v $(foreach d,$(notdir $(DOCS_MAN3)),$(MANDIR)/man3/$(d))
uninstall_doc: uninstall_doc_html uninstall_doc_man

71
configure.ac
View File

@@ -158,27 +158,29 @@ AC_SUBST([abs_objroot])
dnl Munge install path variables.
case "$prefix" in
*\ * ) AC_MSG_ERROR([Prefix should not contain spaces]) ;;
"NONE" ) prefix="/usr/local" ;;
esac
case "$exec_prefix" in
*\ * ) AC_MSG_ERROR([Exec prefix should not contain spaces]) ;;
"NONE" ) exec_prefix=$prefix ;;
esac
PREFIX=$prefix
AC_SUBST([PREFIX])
BINDIR=`eval echo "\"$bindir\""`
BINDIR=`eval echo "\"$BINDIR\""`
BINDIR=`eval echo $bindir`
BINDIR=`eval echo $BINDIR`
AC_SUBST([BINDIR])
INCLUDEDIR=`eval echo "\"$includedir\""`
INCLUDEDIR=`eval echo "\"$INCLUDEDIR\""`
INCLUDEDIR=`eval echo $includedir`
INCLUDEDIR=`eval echo $INCLUDEDIR`
AC_SUBST([INCLUDEDIR])
LIBDIR=`eval echo "\"$libdir\""`
LIBDIR=`eval echo "\"$LIBDIR\""`
LIBDIR=`eval echo $libdir`
LIBDIR=`eval echo $LIBDIR`
AC_SUBST([LIBDIR])
DATADIR=`eval echo "\"$datadir\""`
DATADIR=`eval echo "\"$DATADIR\""`
DATADIR=`eval echo $datadir`
DATADIR=`eval echo $DATADIR`
AC_SUBST([DATADIR])
MANDIR=`eval echo "\"$mandir\""`
MANDIR=`eval echo "\"$MANDIR\""`
MANDIR=`eval echo $mandir`
MANDIR=`eval echo $MANDIR`
AC_SUBST([MANDIR])
dnl Support for building documentation.
@@ -374,53 +376,7 @@ fi
if test "x$enable_cxx" = "x1"; then
AC_DEFINE([JEMALLOC_ENABLE_CXX], [ ], [ ])
fi
if test "x$enable_cxx" = "x1"; then
dnl Now check whether the C++ compiler has exceptions enabled.
AC_LANG_PUSH([C++])
SAVED_CXXFLAGS="${CXXFLAGS}"
CXXFLAGS="${CXXFLAGS} ${EXTRA_CXXFLAGS}"
JE_COMPILABLE([C++ exception support], [
#include <new>
], [
try {
throw std::bad_alloc();
} catch (const std::bad_alloc &) {
}
], [je_cv_cxx_exceptions])
CXXFLAGS="${SAVED_CXXFLAGS}"
AC_LANG_POP([C++])
if test "x${je_cv_cxx_exceptions}" = "xyes" ; then
AC_DEFINE([JEMALLOC_HAVE_CXX_EXCEPTIONS], [ ], [ ])
enable_cxx_exceptions="1"
else
enable_cxx_exceptions="0"
fi
fi
dnl Do not enable infallible-new by default. When enabled, the throwing
dnl operator new aborts on OOM (with size logged) instead of throwing
dnl std::bad_alloc; nothrow new still returns null. With LTO this lets the
dnl compiler prove operator new doesn't throw and elide exception cleanup.
AC_ARG_ENABLE([cxx-infallible-new],
[AS_HELP_STRING([--enable-cxx-infallible-new],
[Abort on OOM in throwing operator new instead of throwing std::bad_alloc])],
[if test "x$enable_cxx_infallible_new" = "xno" ; then
enable_cxx_infallible_new="0"
else
enable_cxx_infallible_new="1"
fi
],
[enable_cxx_infallible_new="0"]
)
if test "x$enable_cxx" != "x1" ; then
enable_cxx_infallible_new="0"
fi
if test "x$enable_cxx_infallible_new" = "x1" ; then
AC_DEFINE([JEMALLOC_INFALLIBLE_NEW], [ ], [ ])
fi
AC_SUBST([enable_cxx])
AC_SUBST([enable_cxx_exceptions])
AC_SUBST([enable_cxx_infallible_new])
AC_SUBST([CONFIGURE_CXXFLAGS])
AC_SUBST([SPECIFIED_CXXFLAGS])
AC_SUBST([EXTRA_CXXFLAGS])
@@ -2085,9 +2041,6 @@ if test "x${LG_PAGE}" != "xundefined" ; then
else
AC_MSG_ERROR([cannot determine value for LG_PAGE])
fi
if test "${LG_PAGE}" -lt 8 ; then
AC_MSG_ERROR([LG_PAGE must be at least 8 (PAGE >= 256)])
fi
AC_ARG_WITH([lg_hugepage],
[AS_HELP_STRING([--with-lg-hugepage=<lg-hugepage>],

100
doc/jemalloc.xml.in
View File

@@ -846,16 +846,6 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
build configuration.</para></listitem>
</varlistentry>
<varlistentry id="config.infallible_new">
<term>
<mallctl>config.infallible_new</mallctl>
(<type>bool</type>)
<literal>r-</literal>
</term>
<listitem><para><option>--enable-cxx-infallible-new</option> was
specified during build configuration.</para></listitem>
</varlistentry>
<varlistentry id="config.lazy_lock">
<term>
<mallctl>config.lazy_lock</mallctl>
@@ -2098,7 +2088,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<parameter>*oldlenp</parameter> must be
<code language="C">sizeof(<type>char *</type>)</code>.
Arena names are also included in the output of <link
linkend="malloc_stats_print_opts"><function>malloc_stats_print()</function></link>.
linkend="stats_print"><function>malloc_stats_print()</function></link>.
</para></listitem>
</varlistentry>
@@ -2182,15 +2172,6 @@ malloc_conf = "xmalloc:true";]]></programlisting>
addition there may be extents created prior to the application having an
opportunity to take over extent allocation.</para>
<para>An extent must be operated on (dalloc, destroy, commit, decommit,
purge, split, merge) by a hook capable of handling it, normally the hook
that allocated it. Replacing hooks on a live arena is tricky and thus
discouraged. If the hook is replaced anyway, the new hook should forward
operations on extents it did not allocate to the previous hook (e.g.,
the new dalloc dispatches to the previous dalloc for an
old-hook-allocated extent). The new hook should also avoid merging
extents allocated by different hooks.</para>
<programlisting language="C"><![CDATA[
typedef extent_hooks_s extent_hooks_t;
struct extent_hooks_s {
@@ -2255,18 +2236,6 @@ struct extent_hooks_s {
linkend="arena.i.dss"><mallctl>arena.&lt;i&gt;.dss</mallctl></link>
setting irrelevant.</para>
<para>The alloc hook may bitwise-OR
<constant>EXTENT_ALLOC_FLAG_PINNED</constant> into the low bits of
the returned pointer to indicate that the backing memory is
non-reclaimable (e.g. HugeTLB pages) and should be reused
preferentially; in that case <parameter>*commit</parameter> must also
be set to true. jemalloc strips the low byte before use. The
pinned attribute is per-extent rather than per-hook: a single alloc
hook may return pinned and non-pinned extents in different calls.
Pinned-ness is set at allocation, inherited through splits, and
never changes after that. Pinned and non-pinned extents are never
merged together.</para>
<funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_dalloc_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
@@ -2488,7 +2457,7 @@ struct extent_hooks_s {
</term>
<listitem><para>Hugepage size. This value is also reported in the
output of <link
linkend="malloc_stats_print_opts"><function>malloc_stats_print()</function></link>.
linkend="stats_print"><function>malloc_stats_print()</function></link>.
</para></listitem>
</varlistentry>
@@ -2799,11 +2768,11 @@ struct extent_hooks_s {
</term>
<listitem><para>Maximum number of bytes in physically resident data
pages mapped by the allocator, comprising all pages dedicated to
allocator metadata, pages backing active allocations, unused dirty
pages, and pinned pages. This is a maximum rather than precise because
pages may not actually be physically resident if they correspond to
demand-zeroed virtual memory that has not yet been touched. This is a
multiple of the page size, and is larger than <link
allocator metadata, pages backing active allocations, and unused dirty
pages. This is a maximum rather than precise because pages may not
actually be physically resident if they correspond to demand-zeroed
virtual memory that has not yet been touched. This is a multiple of the
page size, and is larger than <link
linkend="stats.active"><mallctl>stats.active</mallctl></link>.</para></listitem>
</varlistentry>
@@ -2842,22 +2811,6 @@ struct extent_hooks_s {
</para></listitem>
</varlistentry>
<varlistentry id="stats.pinned">
<term>
<mallctl>stats.pinned</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Total number of bytes in unused extents backed by
non-reclaimable memory. Pinned extents are tracked separately from
dirty, muzzy, and retained extents because they are excluded from
decay and purging; unlike <link
linkend="stats.retained"><mallctl>stats.retained</mallctl></link>,
pinned bytes are included in <link
linkend="stats.mapped"><mallctl>stats.mapped</mallctl></link>.</para></listitem>
</varlistentry>
<varlistentry id="stats.zero_reallocs">
<term>
<mallctl>stats.zero_reallocs</mallctl>
@@ -3136,18 +3089,6 @@ struct extent_hooks_s {
details.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.pinned">
<term>
<mallctl>stats.arenas.&lt;i&gt;.pinned</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Number of pinned bytes. See <link
linkend="stats.pinned"><mallctl>stats.pinned</mallctl></link> for
details.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.extent_avail">
<term>
<mallctl>stats.arenas.&lt;i&gt;.extent_avail</mallctl>
@@ -3205,11 +3146,11 @@ struct extent_hooks_s {
</term>
<listitem><para>Maximum number of bytes in physically resident data
pages mapped by the arena, comprising all pages dedicated to allocator
metadata, pages backing active allocations, unused dirty pages, and
pinned pages. This is a maximum rather than precise because pages
may not actually be physically resident if they correspond to
demand-zeroed virtual memory that has not yet been touched. This is
a multiple of the page size.</para></listitem>
metadata, pages backing active allocations, and unused dirty pages.
This is a maximum rather than precise because pages may not actually be
physically resident if they correspond to demand-zeroed virtual memory
that has not yet been touched. This is a multiple of the page
size.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.dirty_npurge">
@@ -3552,7 +3493,7 @@ struct extent_hooks_s {
</term>
<listitem><para> Number of extents of the given type in this arena in
the bucket corresponding to page size index &lt;j&gt;. The extent type
is one of dirty, muzzy, retained, or pinned.</para></listitem>
is one of dirty, muzzy, or retained.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.extents.bytes">
@@ -3564,7 +3505,7 @@ struct extent_hooks_s {
</term>
<listitem><para> Sum of the bytes managed by extents of the given type
in this arena in the bucket corresponding to page size index &lt;j&gt;.
The extent type is one of dirty, muzzy, retained, or pinned.</para></listitem>
The extent type is one of dirty, muzzy, or retained.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.lextents.j.nmalloc">
@@ -3684,19 +3625,6 @@ struct extent_hooks_s {
counters</link>.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.mutexes.extents_pinned">
<term>
<mallctl>stats.arenas.&lt;i&gt;.mutexes.extents_pinned.{counter}</mallctl>
(<type>counter specific type</type>) <literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Statistics on <varname>arena.&lt;i&gt;.extents_pinned
</varname> mutex (arena scope; pinned extents related).
<mallctl>{counter}</mallctl> is one of the counters in <link
linkend="mutex_counters">mutex profiling
counters</link>.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.mutexes.decay_dirty">
<term>
<mallctl>stats.arenas.&lt;i&gt;.mutexes.decay_dirty.{counter}</mallctl>

293
include/jemalloc/internal/arena.h
View File

@@ -1,293 +0,0 @@
#ifndef JEMALLOC_INTERNAL_ARENA_H
#define JEMALLOC_INTERNAL_ARENA_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena_decay_constants.h"
#include "jemalloc/internal/arena_stats.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bin.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/counter.h"
#include "jemalloc/internal/div.h"
#include "jemalloc/internal/ecache.h"
#include "jemalloc/internal/edata_cache.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/nstime.h"
#include "jemalloc/internal/pa.h"
#include "jemalloc/internal/pages.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/stats.h"
#include "jemalloc/internal/ticker.h"
/******************************************************************************/
/* TYPES */
/******************************************************************************/
/* Default decay times in milliseconds. */
#define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000)
#define MUZZY_DECAY_MS_DEFAULT (0)
/* Maximum length of the arena name. */
#define ARENA_NAME_LEN 32
typedef enum {
percpu_arena_mode_names_base = 0, /* Used for options processing. */
/*
* *_uninit are used only during bootstrapping, and must correspond
* to initialized variant plus percpu_arena_mode_enabled_base.
*/
percpu_arena_uninit = 0,
per_phycpu_arena_uninit = 1,
/* All non-disabled modes must come after percpu_arena_disabled. */
percpu_arena_disabled = 2,
percpu_arena_mode_names_limit = 3, /* Used for options processing. */
percpu_arena_mode_enabled_base = 3,
percpu_arena = 3,
per_phycpu_arena = 4 /* Hyper threads share arena. */
} percpu_arena_mode_t;
#define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base)
#define PERCPU_ARENA_DEFAULT percpu_arena_disabled
/*
* When allocation_size >= oversize_threshold, use the dedicated huge arena
* (unless have explicitly spicified arena index). 0 disables the feature.
*/
#define OVERSIZE_THRESHOLD_DEFAULT (8 << 20)
struct arena_config_s {
/* extent hooks to be used for the arena */
extent_hooks_t *extent_hooks;
/*
* Use extent hooks for metadata (base) allocations when true.
*/
bool metadata_use_hooks;
};
typedef struct arena_config_s arena_config_t;
extern const arena_config_t arena_config_default;
/******************************************************************************/
/* STRUCTS */
/******************************************************************************/
struct arena_s {
/*
* Number of threads currently assigned to this arena. Each thread has
* two distinct assignments, one for application-serving allocation, and
* the other for internal metadata allocation. Internal metadata must
* not be allocated from arenas explicitly created via the arenas.create
* mallctl, because the arena.<i>.reset mallctl indiscriminately
* discards all allocations for the affected arena.
*
* 0: Application allocation.
* 1: Internal metadata allocation.
*
* Synchronization: atomic.
*/
atomic_u_t nthreads[2];
/* Next bin shard for binding new threads. Synchronization: atomic. */
atomic_u_t binshard_next;
/*
* When percpu_arena is enabled, to amortize the cost of reading /
* updating the current CPU id, track the most recent thread accessing
* this arena, and only read CPU if there is a mismatch.
*/
tsdn_t *last_thd;
/* Synchronization: internal. */
arena_stats_t stats;
/*
* List of cache_bin_array_descriptors for extant threads associated
* with this arena. Stats from these are merged incrementally, and at
* exit if opt_stats_print is enabled.
*
* Synchronization: cache_bin_array_descriptor_ql_mtx.
*/
ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql;
malloc_mutex_t cache_bin_array_descriptor_ql_mtx;
/*
* Represents a dss_prec_t, but atomically.
*
* Synchronization: atomic.
*/
atomic_u_t dss_prec;
/*
* Extant large allocations.
*
* Synchronization: large_mtx.
*/
edata_list_active_t large;
/* Synchronizes all large allocation/update/deallocation. */
malloc_mutex_t large_mtx;
/* The page-level allocator shard this arena uses. */
pa_shard_t pa_shard;
/*
* A cached copy of base->ind. This can get accessed on hot paths;
* looking it up in base requires an extra pointer hop / cache miss.
*/
unsigned ind;
/*
* Base allocator, from which arena metadata are allocated.
*
* Synchronization: internal.
*/
base_t *base;
/* Used to determine uptime. Read-only after initialization. */
nstime_t create_time;
/* The name of the arena. */
char name[ARENA_NAME_LEN];
/*
* The arena is allocated alongside its bins; really this is a
* dynamically sized array determined by the binshard settings.
* Enforcing cacheline-alignment to minimize the number of cachelines
* touched on the hot paths.
*/
JEMALLOC_WARN_ON_USAGE(
"Do not use this field directly. "
"Use `arena_get_bin` instead.")
JEMALLOC_ALIGNED(CACHELINE)
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
bin_t all_bins[];
#else
bin_t all_bins[0];
#endif
};
/******************************************************************************/
/* EXTERNS */
/******************************************************************************/
/*
* When the amount of pages to be purged exceeds this amount, deferred purge
* should happen.
*/
#define ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD UINT64_C(1024)
extern ssize_t opt_dirty_decay_ms;
extern ssize_t opt_muzzy_decay_ms;
extern percpu_arena_mode_t opt_percpu_arena;
extern const char *const percpu_arena_mode_names[];
extern div_info_t arena_binind_div_info[SC_NBINS];
extern emap_t arena_emap_global;
extern size_t opt_oversize_threshold;
extern size_t oversize_threshold;
extern bool opt_huge_arena_pac_thp;
extern pac_thp_t huge_arena_pac_thp;
/*
* arena_bin_offsets[binind] is the offset of the first bin shard for size class
* binind.
*/
extern uint32_t arena_bin_offsets[SC_NBINS];
void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
size_t *nactive, size_t *ndirty, size_t *nmuzzy);
void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
hpa_shard_stats_t *hpastats);
void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
edata_t *arena_extent_alloc_large(
tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
void arena_extent_dalloc_large_prep(
tsdn_t *tsdn, arena_t *arena, const edata_t *edata);
void arena_extent_ralloc_large_shrink(
tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
void arena_extent_ralloc_large_expand(
tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
bool arena_decay_ms_set(
tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms);
ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state);
void arena_decay(
tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all);
uint64_t arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena);
void arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
void arena_reset(tsd_t *tsd, arena_t *arena);
void arena_destroy(tsd_t *tsd, arena_t *arena);
cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,
szind_t binind, cache_bin_ptr_array_t *arr, const cache_bin_sz_t nfill_min,
const cache_bin_sz_t nfill_max, cache_bin_stats_t merge_stats);
void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
bool zero, bool slab);
void arena_prof_promote(
tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
size_t arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr);
void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
void arena_dalloc_small(tsdn_t *tsdn, void *ptr);
void arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
cache_bin_ptr_array_t *arr, unsigned nflush, bool small,
arena_t *stats_arena, cache_bin_stats_t merge_stats);
bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero, size_t *newsize);
dss_prec_t arena_dss_prec_get(const arena_t *arena);
ehooks_t *arena_get_ehooks(const arena_t *arena);
extent_hooks_t *arena_set_extent_hooks(
tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks);
bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
void arena_name_get(const arena_t *arena, char *name);
void arena_name_set(arena_t *arena, const char *name);
ssize_t arena_dirty_decay_ms_default_get(void);
bool arena_dirty_decay_ms_default_set(ssize_t decay_ms);
ssize_t arena_muzzy_decay_ms_default_get(void);
bool arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
bool arena_retain_grow_limit_get_set(
tsd_t *tsd, arena_t *arena, size_t *old_limit, size_t *new_limit);
unsigned arena_nthreads_get(const arena_t *arena, bool internal);
void arena_nthreads_inc(arena_t *arena, bool internal);
void arena_nthreads_dec(arena_t *arena, bool internal);
arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
bool arena_init_huge(tsdn_t *tsdn, arena_t *a0);
bool arena_ind_is_huge(unsigned ind);
arena_t *arena_choose_huge(tsd_t *tsd);
bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
void *arena_locality_hint(tsdn_t *tsdn, arena_t *arena, szind_t szind);
void arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena,
cache_bin_array_descriptor_t *desc);
void arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
cache_bin_array_descriptor_t *desc);
void arena_cache_bin_array_postfork_child(arena_t *arena,
cache_bin_array_descriptor_t *desc_or_null);
void arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena);
void arena_prefork0(tsdn_t *tsdn, arena_t *arena);
void arena_prefork1(tsdn_t *tsdn, arena_t *arena);
void arena_prefork2(tsdn_t *tsdn, arena_t *arena);
void arena_prefork3(tsdn_t *tsdn, arena_t *arena);
void arena_prefork4(tsdn_t *tsdn, arena_t *arena);
void arena_prefork5(tsdn_t *tsdn, arena_t *arena);
void arena_prefork6(tsdn_t *tsdn, arena_t *arena);
void arena_prefork7(tsdn_t *tsdn, arena_t *arena);
void arena_prefork8(tsdn_t *tsdn, arena_t *arena);
void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
void arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
cache_bin_array_descriptor_t *surviving_desc);
#endif /* JEMALLOC_INTERNAL_ARENA_H */

13
include/jemalloc/internal/arena_decay_constants.h
View File

@@ -1,13 +0,0 @@
#ifndef JEMALLOC_INTERNAL_ARENA_DECAY_CONSTANTS_H
#define JEMALLOC_INTERNAL_ARENA_DECAY_CONSTANTS_H
/*
* Minimal header so both arena.h and tsd_internals.h can share decay-related
* constants without dragging the full arena types into the tsd parse chain
* (which is loaded long before arena.h via ckh.h -> tsd.h).
*/
/* Number of event ticks between time checks. */
#define ARENA_DECAY_NTICKS_PER_UPDATE 1000
#endif /* JEMALLOC_INTERNAL_ARENA_DECAY_CONSTANTS_H */

125
include/jemalloc/internal/arena_externs.h Normal file
View File

@@ -0,0 +1,125 @@
#ifndef JEMALLOC_INTERNAL_ARENA_EXTERNS_H
#define JEMALLOC_INTERNAL_ARENA_EXTERNS_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena_stats.h"
#include "jemalloc/internal/bin.h"
#include "jemalloc/internal/div.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/hook.h"
#include "jemalloc/internal/pages.h"
#include "jemalloc/internal/stats.h"
/*
* When the amount of pages to be purged exceeds this amount, deferred purge
* should happen.
*/
#define ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD UINT64_C(1024)
extern ssize_t opt_dirty_decay_ms;
extern ssize_t opt_muzzy_decay_ms;
extern percpu_arena_mode_t opt_percpu_arena;
extern const char *const percpu_arena_mode_names[];
extern div_info_t arena_binind_div_info[SC_NBINS];
extern emap_t arena_emap_global;
extern size_t opt_oversize_threshold;
extern size_t oversize_threshold;
extern bool opt_huge_arena_pac_thp;
extern pac_thp_t huge_arena_pac_thp;
/*
* arena_bin_offsets[binind] is the offset of the first bin shard for size class
* binind.
*/
extern uint32_t arena_bin_offsets[SC_NBINS];
void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
size_t *nactive, size_t *ndirty, size_t *nmuzzy);
void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
hpa_shard_stats_t *hpastats);
void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
edata_t *arena_extent_alloc_large(
tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
void arena_extent_dalloc_large_prep(
tsdn_t *tsdn, arena_t *arena, edata_t *edata);
void arena_extent_ralloc_large_shrink(
tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize);
void arena_extent_ralloc_large_expand(
tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize);
bool arena_decay_ms_set(
tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms);
ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state);
void arena_decay(
tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all);
uint64_t arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena);
void arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
void arena_reset(tsd_t *tsd, arena_t *arena);
void arena_destroy(tsd_t *tsd, arena_t *arena);
cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,
szind_t binind, cache_bin_ptr_array_t *arr, const cache_bin_sz_t nfill_min,
const cache_bin_sz_t nfill_max, cache_bin_stats_t merge_stats);
void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
bool zero, bool slab);
void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
bool zero, bool slab, tcache_t *tcache);
void arena_prof_promote(
tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
void arena_dalloc_promoted(
tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
void arena_dalloc_small(tsdn_t *tsdn, void *ptr);
void arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
cache_bin_ptr_array_t *arr, unsigned nflush, bool small,
arena_t *stats_arena, cache_bin_stats_t merge_stats);
bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero, size_t *newsize);
void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache,
hook_ralloc_args_t *hook_args);
dss_prec_t arena_dss_prec_get(arena_t *arena);
ehooks_t *arena_get_ehooks(arena_t *arena);
extent_hooks_t *arena_set_extent_hooks(
tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks);
bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
void arena_name_get(arena_t *arena, char *name);
void arena_name_set(arena_t *arena, const char *name);
ssize_t arena_dirty_decay_ms_default_get(void);
bool arena_dirty_decay_ms_default_set(ssize_t decay_ms);
ssize_t arena_muzzy_decay_ms_default_get(void);
bool arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
bool arena_retain_grow_limit_get_set(
tsd_t *tsd, arena_t *arena, size_t *old_limit, size_t *new_limit);
unsigned arena_nthreads_get(arena_t *arena, bool internal);
void arena_nthreads_inc(arena_t *arena, bool internal);
void arena_nthreads_dec(arena_t *arena, bool internal);
arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
bool arena_init_huge(tsdn_t *tsdn, arena_t *a0);
arena_t *arena_choose_huge(tsd_t *tsd);
size_t arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
void **ptrs, size_t nfill, bool zero);
bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
void arena_prefork0(tsdn_t *tsdn, arena_t *arena);
void arena_prefork1(tsdn_t *tsdn, arena_t *arena);
void arena_prefork2(tsdn_t *tsdn, arena_t *arena);
void arena_prefork3(tsdn_t *tsdn, arena_t *arena);
void arena_prefork4(tsdn_t *tsdn, arena_t *arena);
void arena_prefork5(tsdn_t *tsdn, arena_t *arena);
void arena_prefork6(tsdn_t *tsdn, arena_t *arena);
void arena_prefork7(tsdn_t *tsdn, arena_t *arena);
void arena_prefork8(tsdn_t *tsdn, arena_t *arena);
void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
void arena_postfork_child(tsdn_t *tsdn, arena_t *arena);
#endif /* JEMALLOC_INTERNAL_ARENA_EXTERNS_H */

400
include/jemalloc/internal/arena_inlines.h
View File

@@ -1,400 +0,0 @@
#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_H
#define JEMALLOC_INTERNAL_ARENA_INLINES_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/arenas_management.h"
#include "jemalloc/internal/bin_inlines.h"
#include "jemalloc/internal/div.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/extent.h"
#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/large.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/prof.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/safety_check.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/ticker.h"
/* Cheap field accessors. */
static inline unsigned
arena_ind_get(const arena_t *arena) {
return arena->ind;
}
static inline void
arena_internal_add(arena_t *arena, size_t size) {
atomic_fetch_add_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
}
static inline void
arena_internal_sub(arena_t *arena, size_t size) {
atomic_fetch_sub_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
}
static inline size_t
arena_internal_get(const arena_t *arena) {
return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED);
}
static inline bool
arena_is_auto(const arena_t *arena) {
assert(narenas_auto > 0);
return (arena_ind_get(arena) < manual_arena_base);
}
static inline arena_t *
arena_get_from_edata(const edata_t *edata) {
return (arena_t *)atomic_load_p(
&arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
}
/* Arena selection and migration. */
static inline void
thread_migrate_arena(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
assert(oldarena != NULL);
assert(newarena != NULL);
arena_migrate(tsd, oldarena, newarena);
if (tcache_available(tsd)) {
tcache_arena_reassociate(tsd_tsdn(tsd),
tsd_tcache_slowp_get(tsd), newarena);
}
}
static inline void
percpu_arena_update(tsd_t *tsd, unsigned cpu) {
assert(have_percpu_arena);
arena_t *oldarena = tsd_arena_get(tsd);
assert(oldarena != NULL);
unsigned oldind = arena_ind_get(oldarena);
if (oldind != cpu) {
unsigned newind = cpu;
arena_t *newarena = arena_get(tsd_tsdn(tsd), newind, true);
assert(newarena != NULL);
thread_migrate_arena(tsd, oldarena, newarena);
}
}
/* Choose an arena based on a per-thread value. */
static inline arena_t *
arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
arena_t *ret;
if (arena != NULL) {
return arena;
}
/* During reentrancy, arena 0 is the safest bet. */
if (unlikely(tsd_reentrancy_level_get(tsd) > 0)) {
return arena_get(tsd_tsdn(tsd), 0, true);
}
ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd);
if (unlikely(ret == NULL)) {
ret = arena_choose_hard(tsd, internal);
assert(ret);
if (tcache_available(tsd)) {
tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
if (tcache_slow->arena != NULL) {
/* See comments in tsd_tcache_data_init().*/
assert(tcache_slow->arena
== arena_get(tsd_tsdn(tsd), 0, false));
if (tcache_slow->arena != ret) {
tcache_arena_reassociate(tsd_tsdn(tsd),
tcache_slow, ret);
}
} else {
tcache_arena_associate(
tsd_tsdn(tsd), tcache_slow, ret);
}
}
}
/*
* Note that for percpu arena, if the current arena is outside of the
* auto percpu arena range, (i.e. thread is assigned to a manually
* managed arena), then percpu arena is skipped.
*/
if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)
&& !internal
&& (arena_ind_get(ret) < percpu_arena_ind_limit(opt_percpu_arena))
&& (ret->last_thd != tsd_tsdn(tsd))) {
unsigned ind = percpu_arena_choose();
if (arena_ind_get(ret) != ind) {
percpu_arena_update(tsd, ind);
ret = tsd_arena_get(tsd);
}
ret->last_thd = tsd_tsdn(tsd);
}
return ret;
}
static inline arena_t *
arena_choose(tsd_t *tsd, arena_t *arena) {
return arena_choose_impl(tsd, arena, false);
}
static inline arena_t *
arena_ichoose(tsd_t *tsd, arena_t *arena) {
return arena_choose_impl(tsd, arena, true);
}
JEMALLOC_ALWAYS_INLINE arena_t *
arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
if (arena != NULL) {
return arena;
}
/*
* For huge allocations, use the dedicated huge arena if both are true:
* 1) is using auto arena selection (i.e. arena == NULL), and 2) the
* thread is not assigned to a manual arena.
*/
arena_t *tsd_arena = tsd_arena_get(tsd);
if (tsd_arena == NULL) {
tsd_arena = arena_choose(tsd, NULL);
}
size_t threshold = atomic_load_zu(
&tsd_arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED);
if (unlikely(size >= threshold) && arena_is_auto(tsd_arena)) {
return arena_choose_huge(tsd);
}
return tsd_arena;
}
JEMALLOC_ALWAYS_INLINE bool
large_dalloc_safety_checks(edata_t *edata, const void *ptr, size_t input_size) {
if (!config_opt_safety_checks) {
return false;
}
/*
* Eagerly detect double free and sized dealloc bugs for large sizes.
* The cost is low enough (as edata will be accessed anyway) to be
* enabled all the time.
*/
if (unlikely(edata == NULL
|| edata_state_get(edata) != extent_state_active)) {
safety_check_fail(
"Invalid deallocation detected: "
"pages being freed (%p) not currently active, "
"possibly caused by double free bugs.",
ptr);
return true;
}
if (unlikely(input_size != edata_usize_get(edata)
|| input_size > SC_LARGE_MAXCLASS)) {
safety_check_fail_sized_dealloc(/* current_dealloc */ true, ptr,
/* true_size */ edata_usize_get(edata), input_size);
return true;
}
return false;
}
JEMALLOC_ALWAYS_INLINE void
arena_prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx,
prof_info_t *prof_info, bool reset_recent) {
cassert(config_prof);
assert(ptr != NULL);
assert(prof_info != NULL);
edata_t *edata = NULL;
bool is_slab;
/* Static check. */
if (alloc_ctx == NULL) {
edata = emap_edata_lookup(
tsd_tsdn(tsd), &arena_emap_global, ptr);
is_slab = edata_slab_get(edata);
} else if (unlikely(!(is_slab = alloc_ctx->slab))) {
edata = emap_edata_lookup(
tsd_tsdn(tsd), &arena_emap_global, ptr);
}
if (unlikely(!is_slab)) {
/* edata must have been initialized at this point. */
assert(edata != NULL);
size_t usize = (alloc_ctx == NULL)
? edata_usize_get(edata)
: emap_alloc_ctx_usize_get(alloc_ctx);
if (reset_recent
&& large_dalloc_safety_checks(edata, ptr, usize)) {
prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
return;
}
large_prof_info_get(tsd, edata, prof_info, reset_recent);
} else {
prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
/*
* No need to set other fields in prof_info; they will never be
* accessed if alloc_tctx == PROF_TCTX_SENTINEL.
*/
}
}
JEMALLOC_ALWAYS_INLINE void
arena_prof_tctx_reset(
tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
cassert(config_prof);
assert(ptr != NULL);
/* Static check. */
if (alloc_ctx == NULL) {
edata_t *edata = emap_edata_lookup(
tsd_tsdn(tsd), &arena_emap_global, ptr);
if (unlikely(!edata_slab_get(edata))) {
large_prof_tctx_reset(edata);
}
} else {
if (unlikely(!alloc_ctx->slab)) {
edata_t *edata = emap_edata_lookup(
tsd_tsdn(tsd), &arena_emap_global, ptr);
large_prof_tctx_reset(edata);
}
}
}
JEMALLOC_ALWAYS_INLINE void
arena_prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) {
cassert(config_prof);
assert(ptr != NULL);
edata_t *edata = emap_edata_lookup(
tsd_tsdn(tsd), &arena_emap_global, ptr);
assert(!edata_slab_get(edata));
large_prof_tctx_reset(edata);
}
JEMALLOC_ALWAYS_INLINE void
arena_prof_info_set(
tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) {
cassert(config_prof);
assert(!edata_slab_get(edata));
large_prof_info_set(edata, tctx, size);
}
JEMALLOC_ALWAYS_INLINE void
arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
if (unlikely(tsdn_null(tsdn))) {
return;
}
tsd_t *tsd = tsdn_tsd(tsdn);
/*
* We use the ticker_geom_t to avoid having per-arena state in the tsd.
* Instead of having a countdown-until-decay timer running for every
* arena in every thread, we flip a coin once per tick, whose
* probability of coming up heads is 1/nticks; this is effectively the
* operation of the ticker_geom_t. Each arena has the same chance of a
* coinflip coming up heads (1/ARENA_DECAY_NTICKS_PER_UPDATE), so we can
* use a single ticker for all of them.
*/
ticker_geom_t *decay_ticker = tsd_arena_decay_tickerp_get(tsd);
uint64_t *prng_state = tsd_prng_statep_get(tsd);
if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks,
tsd_reentrancy_level_get(tsd) > 0))) {
arena_decay(tsdn, arena, false, false);
}
}
JEMALLOC_ALWAYS_INLINE void
arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
arena_decay_ticks(tsdn, arena, 1);
}
JEMALLOC_ALWAYS_INLINE arena_t *
arena_aalloc(tsdn_t *tsdn, const void *ptr) {
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
unsigned arena_ind = edata_arena_ind_get(edata);
return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_RELAXED);
}
JEMALLOC_ALWAYS_INLINE size_t
arena_salloc(tsdn_t *tsdn, const void *ptr) {
assert(ptr != NULL);
emap_alloc_ctx_t alloc_ctx;
emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
assert(alloc_ctx.szind != SC_NSIZES);
return emap_alloc_ctx_usize_get(&alloc_ctx);
}
JEMALLOC_ALWAYS_INLINE size_t
arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
/*
* Return 0 if ptr is not within an extent managed by jemalloc. This
* function has two extra costs relative to isalloc():
* - The rtree calls cannot claim to be dependent lookups, which induces
* rtree lookup load dependencies.
* - The lookup may fail, so there is an extra branch to check for
* failure.
*/
emap_full_alloc_ctx_t full_alloc_ctx;
bool missing = emap_full_alloc_ctx_try_lookup(
tsdn, &arena_emap_global, ptr, &full_alloc_ctx);
if (missing) {
return 0;
}
if (full_alloc_ctx.edata == NULL) {
return 0;
}
assert(edata_state_get(full_alloc_ctx.edata) == extent_state_active);
/* Only slab members should be looked up via interior pointers. */
assert(edata_addr_get(full_alloc_ctx.edata) == ptr
|| edata_slab_get(full_alloc_ctx.edata));
assert(full_alloc_ctx.szind != SC_NSIZES);
return edata_usize_get(full_alloc_ctx.edata);
}
static inline void
arena_cache_oblivious_randomize(
tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) {
assert(edata_base_get(edata) == edata_addr_get(edata));
if (alignment < PAGE) {
unsigned lg_range = LG_PAGE
- lg_floor(CACHELINE_CEILING(alignment));
size_t r;
if (!tsdn_null(tsdn)) {
tsd_t *tsd = tsdn_tsd(tsdn);
r = (size_t)prng_lg_range_u64(
tsd_prng_statep_get(tsd), lg_range);
} else {
uint64_t stack_value = (uint64_t)(uintptr_t)&r;
r = (size_t)prng_lg_range_u64(&stack_value, lg_range);
}
uintptr_t random_offset = ((uintptr_t)r)
<< (LG_PAGE - lg_range);
edata->e_addr = (void *)((byte_t *)edata->e_addr
+ random_offset);
assert(ALIGNMENT_ADDR2BASE(edata->e_addr, alignment)
== edata->e_addr);
}
}
static inline bin_t *
arena_get_bin(arena_t *arena, szind_t binind, unsigned binshard) {
bin_t *shard0 = (bin_t *)((byte_t *)arena + arena_bin_offsets[binind]);
return shard0 + binshard;
}
#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_H */

27
include/jemalloc/internal/arena_inlines_a.h Normal file
View File

@@ -0,0 +1,27 @@
#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_A_H
#define JEMALLOC_INTERNAL_ARENA_INLINES_A_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena_structs.h"
static inline unsigned
arena_ind_get(const arena_t *arena) {
return arena->ind;
}
static inline void
arena_internal_add(arena_t *arena, size_t size) {
atomic_fetch_add_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
}
static inline void
arena_internal_sub(arena_t *arena, size_t size) {
atomic_fetch_sub_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
}
static inline size_t
arena_internal_get(arena_t *arena) {
return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED);
}
#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_A_H */

538
include/jemalloc/internal/arena_inlines_b.h Normal file
View File

@@ -0,0 +1,538 @@
#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_B_H
#define JEMALLOC_INTERNAL_ARENA_INLINES_B_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena_externs.h"
#include "jemalloc/internal/arena_structs.h"
#include "jemalloc/internal/bin_inlines.h"
#include "jemalloc/internal/div.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/large_externs.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/prof_externs.h"
#include "jemalloc/internal/prof_structs.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/safety_check.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/tcache_inlines.h"
#include "jemalloc/internal/ticker.h"
static inline arena_t *
arena_get_from_edata(edata_t *edata) {
return (arena_t *)atomic_load_p(
&arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
}
JEMALLOC_ALWAYS_INLINE arena_t *
arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
if (arena != NULL) {
return arena;
}
/*
* For huge allocations, use the dedicated huge arena if both are true:
* 1) is using auto arena selection (i.e. arena == NULL), and 2) the
* thread is not assigned to a manual arena.
*/
arena_t *tsd_arena = tsd_arena_get(tsd);
if (tsd_arena == NULL) {
tsd_arena = arena_choose(tsd, NULL);
}
size_t threshold = atomic_load_zu(
&tsd_arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED);
if (unlikely(size >= threshold) && arena_is_auto(tsd_arena)) {
return arena_choose_huge(tsd);
}
return tsd_arena;
}
JEMALLOC_ALWAYS_INLINE bool
large_dalloc_safety_checks(edata_t *edata, const void *ptr, size_t input_size) {
if (!config_opt_safety_checks) {
return false;
}
/*
* Eagerly detect double free and sized dealloc bugs for large sizes.
* The cost is low enough (as edata will be accessed anyway) to be
* enabled all the time.
*/
if (unlikely(edata == NULL
|| edata_state_get(edata) != extent_state_active)) {
safety_check_fail(
"Invalid deallocation detected: "
"pages being freed (%p) not currently active, "
"possibly caused by double free bugs.",
ptr);
return true;
}
if (unlikely(input_size != edata_usize_get(edata)
|| input_size > SC_LARGE_MAXCLASS)) {
safety_check_fail_sized_dealloc(/* current_dealloc */ true, ptr,
/* true_size */ edata_usize_get(edata), input_size);
return true;
}
return false;
}
JEMALLOC_ALWAYS_INLINE void
arena_prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx,
prof_info_t *prof_info, bool reset_recent) {
cassert(config_prof);
assert(ptr != NULL);
assert(prof_info != NULL);
edata_t *edata = NULL;
bool is_slab;
/* Static check. */
if (alloc_ctx == NULL) {
edata = emap_edata_lookup(
tsd_tsdn(tsd), &arena_emap_global, ptr);
is_slab = edata_slab_get(edata);
} else if (unlikely(!(is_slab = alloc_ctx->slab))) {
edata = emap_edata_lookup(
tsd_tsdn(tsd), &arena_emap_global, ptr);
}
if (unlikely(!is_slab)) {
/* edata must have been initialized at this point. */
assert(edata != NULL);
size_t usize = (alloc_ctx == NULL)
? edata_usize_get(edata)
: emap_alloc_ctx_usize_get(alloc_ctx);
if (reset_recent
&& large_dalloc_safety_checks(edata, ptr, usize)) {
prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
return;
}
large_prof_info_get(tsd, edata, prof_info, reset_recent);
} else {
prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
/*
* No need to set other fields in prof_info; they will never be
* accessed if alloc_tctx == PROF_TCTX_SENTINEL.
*/
}
}
JEMALLOC_ALWAYS_INLINE void
arena_prof_tctx_reset(
tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
cassert(config_prof);
assert(ptr != NULL);
/* Static check. */
if (alloc_ctx == NULL) {
edata_t *edata = emap_edata_lookup(
tsd_tsdn(tsd), &arena_emap_global, ptr);
if (unlikely(!edata_slab_get(edata))) {
large_prof_tctx_reset(edata);
}
} else {
if (unlikely(!alloc_ctx->slab)) {
edata_t *edata = emap_edata_lookup(
tsd_tsdn(tsd), &arena_emap_global, ptr);
large_prof_tctx_reset(edata);
}
}
}
JEMALLOC_ALWAYS_INLINE void
arena_prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) {
cassert(config_prof);
assert(ptr != NULL);
edata_t *edata = emap_edata_lookup(
tsd_tsdn(tsd), &arena_emap_global, ptr);
assert(!edata_slab_get(edata));
large_prof_tctx_reset(edata);
}
JEMALLOC_ALWAYS_INLINE void
arena_prof_info_set(
tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) {
cassert(config_prof);
assert(!edata_slab_get(edata));
large_prof_info_set(edata, tctx, size);
}
JEMALLOC_ALWAYS_INLINE void
arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
if (unlikely(tsdn_null(tsdn))) {
return;
}
tsd_t *tsd = tsdn_tsd(tsdn);
/*
* We use the ticker_geom_t to avoid having per-arena state in the tsd.
* Instead of having a countdown-until-decay timer running for every
* arena in every thread, we flip a coin once per tick, whose
* probability of coming up heads is 1/nticks; this is effectively the
* operation of the ticker_geom_t. Each arena has the same chance of a
* coinflip coming up heads (1/ARENA_DECAY_NTICKS_PER_UPDATE), so we can
* use a single ticker for all of them.
*/
ticker_geom_t *decay_ticker = tsd_arena_decay_tickerp_get(tsd);
uint64_t *prng_state = tsd_prng_statep_get(tsd);
if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks,
tsd_reentrancy_level_get(tsd) > 0))) {
arena_decay(tsdn, arena, false, false);
}
}
JEMALLOC_ALWAYS_INLINE void
arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
arena_decay_ticks(tsdn, arena, 1);
}
JEMALLOC_ALWAYS_INLINE void *
arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
bool slab, tcache_t *tcache, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
if (likely(tcache != NULL)) {
if (likely(slab)) {
assert(sz_can_use_slab(size));
return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
size, ind, zero, slow_path);
} else if (likely(ind < tcache_nbins_get(tcache->tcache_slow)
&& !tcache_bin_disabled(ind, &tcache->bins[ind],
tcache->tcache_slow))) {
return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
size, ind, zero, slow_path);
}
/* (size > tcache_max) case falls through. */
}
return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
}
JEMALLOC_ALWAYS_INLINE arena_t *
arena_aalloc(tsdn_t *tsdn, const void *ptr) {
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
unsigned arena_ind = edata_arena_ind_get(edata);
return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_RELAXED);
}
JEMALLOC_ALWAYS_INLINE size_t
arena_salloc(tsdn_t *tsdn, const void *ptr) {
assert(ptr != NULL);
emap_alloc_ctx_t alloc_ctx;
emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
assert(alloc_ctx.szind != SC_NSIZES);
return emap_alloc_ctx_usize_get(&alloc_ctx);
}
JEMALLOC_ALWAYS_INLINE size_t
arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
/*
* Return 0 if ptr is not within an extent managed by jemalloc. This
* function has two extra costs relative to isalloc():
* - The rtree calls cannot claim to be dependent lookups, which induces
* rtree lookup load dependencies.
* - The lookup may fail, so there is an extra branch to check for
* failure.
*/
emap_full_alloc_ctx_t full_alloc_ctx;
bool missing = emap_full_alloc_ctx_try_lookup(
tsdn, &arena_emap_global, ptr, &full_alloc_ctx);
if (missing) {
return 0;
}
if (full_alloc_ctx.edata == NULL) {
return 0;
}
assert(edata_state_get(full_alloc_ctx.edata) == extent_state_active);
/* Only slab members should be looked up via interior pointers. */
assert(edata_addr_get(full_alloc_ctx.edata) == ptr
|| edata_slab_get(full_alloc_ctx.edata));
assert(full_alloc_ctx.szind != SC_NSIZES);
return edata_usize_get(full_alloc_ctx.edata);
}
static inline void
arena_dalloc_large_no_tcache(
tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
/*
* szind is still needed in this function mainly becuase
* szind < SC_NBINS determines not only if this is a small alloc,
* but also if szind is valid (an inactive extent would have
* szind == SC_NSIZES).
*/
if (config_prof && unlikely(szind < SC_NBINS)) {
arena_dalloc_promoted(tsdn, ptr, NULL, true);
} else {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
if (large_dalloc_safety_checks(edata, ptr, usize)) {
/* See the comment in isfree. */
return;
}
large_dalloc(tsdn, edata);
}
}
static inline void
arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
assert(ptr != NULL);
emap_alloc_ctx_t alloc_ctx;
emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.szind < SC_NSIZES);
assert(alloc_ctx.slab == edata_slab_get(edata));
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
== edata_usize_get(edata));
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
arena_dalloc_small(tsdn, ptr);
} else {
arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx));
}
}
JEMALLOC_ALWAYS_INLINE void
arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
size_t usize, bool slow_path) {
assert(!tsdn_null(tsdn) && tcache != NULL);
bool is_sample_promoted = config_prof && szind < SC_NBINS;
if (unlikely(is_sample_promoted)) {
arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
} else {
if (szind < tcache_nbins_get(tcache->tcache_slow)
&& !tcache_bin_disabled(
szind, &tcache->bins[szind], tcache->tcache_slow)) {
tcache_dalloc_large(
tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
} else {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
if (large_dalloc_safety_checks(edata, ptr, usize)) {
/* See the comment in isfree. */
return;
}
large_dalloc(tsdn, edata);
}
}
}
JEMALLOC_ALWAYS_INLINE bool
arena_tcache_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
if (!config_debug) {
return false;
}
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
szind_t binind = edata_szind_get(edata);
div_info_t div_info = arena_binind_div_info[binind];
/*
* Calls the internal function bin_slab_regind_impl because the
* safety check does not require a lock.
*/
size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
slab_data_t *slab_data = edata_slab_data_get(edata);
const bin_info_t *bin_info = &bin_infos[binind];
assert(edata_nfree_get(edata) < bin_info->nregs);
if (unlikely(!bitmap_get(
slab_data->bitmap, &bin_info->bitmap_info, regind))) {
safety_check_fail(
"Invalid deallocation detected: the pointer being freed (%p) not "
"currently active, possibly caused by double free bugs.\n",
ptr);
return true;
}
return false;
}
JEMALLOC_ALWAYS_INLINE void
arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL);
if (unlikely(tcache == NULL)) {
arena_dalloc_no_tcache(tsdn, ptr);
return;
}
emap_alloc_ctx_t alloc_ctx;
if (caller_alloc_ctx != NULL) {
alloc_ctx = *caller_alloc_ctx;
} else {
util_assume(tsdn != NULL);
emap_alloc_ctx_lookup(
tsdn, &arena_emap_global, ptr, &alloc_ctx);
}
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.szind < SC_NSIZES);
assert(alloc_ctx.slab == edata_slab_get(edata));
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
== edata_usize_get(edata));
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
return;
}
tcache_dalloc_small(
tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
} else {
arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
}
}
static inline void
arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
assert(ptr != NULL);
assert(size <= SC_LARGE_MAXCLASS);
emap_alloc_ctx_t alloc_ctx;
if (!config_prof || !opt_prof) {
/*
* There is no risk of being confused by a promoted sampled
* object, so base szind and slab on the given size.
*/
szind_t szind = sz_size2index(size);
emap_alloc_ctx_init(
&alloc_ctx, szind, (szind < SC_NBINS), size);
}
if ((config_prof && opt_prof) || config_debug) {
emap_alloc_ctx_lookup(
tsdn, &arena_emap_global, ptr, &alloc_ctx);
assert(alloc_ctx.szind == sz_size2index(size));
assert((config_prof && opt_prof)
|| alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.slab == edata_slab_get(edata));
}
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
arena_dalloc_small(tsdn, ptr);
} else {
arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx));
}
}
JEMALLOC_ALWAYS_INLINE void
arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL);
assert(size <= SC_LARGE_MAXCLASS);
if (unlikely(tcache == NULL)) {
arena_sdalloc_no_tcache(tsdn, ptr, size);
return;
}
emap_alloc_ctx_t alloc_ctx;
if (config_prof && opt_prof) {
if (caller_alloc_ctx == NULL) {
/* Uncommon case and should be a static check. */
emap_alloc_ctx_lookup(
tsdn, &arena_emap_global, ptr, &alloc_ctx);
assert(alloc_ctx.szind == sz_size2index(size));
assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
} else {
alloc_ctx = *caller_alloc_ctx;
}
} else {
/*
* There is no risk of being confused by a promoted sampled
* object, so base szind and slab on the given size.
*/
alloc_ctx.szind = sz_size2index(size);
alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
}
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.slab == edata_slab_get(edata));
emap_alloc_ctx_init(
&alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
== edata_usize_get(edata));
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
return;
}
tcache_dalloc_small(
tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
} else {
arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
sz_s2u(size), slow_path);
}
}
static inline void
arena_cache_oblivious_randomize(
tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) {
assert(edata_base_get(edata) == edata_addr_get(edata));
if (alignment < PAGE) {
unsigned lg_range = LG_PAGE
- lg_floor(CACHELINE_CEILING(alignment));
size_t r;
if (!tsdn_null(tsdn)) {
tsd_t *tsd = tsdn_tsd(tsdn);
r = (size_t)prng_lg_range_u64(
tsd_prng_statep_get(tsd), lg_range);
} else {
uint64_t stack_value = (uint64_t)(uintptr_t)&r;
r = (size_t)prng_lg_range_u64(&stack_value, lg_range);
}
uintptr_t random_offset = ((uintptr_t)r)
<< (LG_PAGE - lg_range);
edata->e_addr = (void *)((byte_t *)edata->e_addr
+ random_offset);
assert(ALIGNMENT_ADDR2BASE(edata->e_addr, alignment)
== edata->e_addr);
}
}
static inline bin_t *
arena_get_bin(arena_t *arena, szind_t binind, unsigned binshard) {
bin_t *shard0 = (bin_t *)((byte_t *)arena + arena_bin_offsets[binind]);
return shard0 + binshard;
}
#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_B_H */

111
include/jemalloc/internal/arena_structs.h Normal file
View File

@@ -0,0 +1,111 @@
#ifndef JEMALLOC_INTERNAL_ARENA_STRUCTS_H
#define JEMALLOC_INTERNAL_ARENA_STRUCTS_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena_stats.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bin.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/counter.h"
#include "jemalloc/internal/ecache.h"
#include "jemalloc/internal/edata_cache.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/nstime.h"
#include "jemalloc/internal/pa.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/ticker.h"
struct arena_s {
/*
* Number of threads currently assigned to this arena. Each thread has
* two distinct assignments, one for application-serving allocation, and
* the other for internal metadata allocation. Internal metadata must
* not be allocated from arenas explicitly created via the arenas.create
* mallctl, because the arena.<i>.reset mallctl indiscriminately
* discards all allocations for the affected arena.
*
* 0: Application allocation.
* 1: Internal metadata allocation.
*
* Synchronization: atomic.
*/
atomic_u_t nthreads[2];
/* Next bin shard for binding new threads. Synchronization: atomic. */
atomic_u_t binshard_next;
/*
* When percpu_arena is enabled, to amortize the cost of reading /
* updating the current CPU id, track the most recent thread accessing
* this arena, and only read CPU if there is a mismatch.
*/
tsdn_t *last_thd;
/* Synchronization: internal. */
arena_stats_t stats;
/*
* Lists of tcaches and cache_bin_array_descriptors for extant threads
* associated with this arena. Stats from these are merged
* incrementally, and at exit if opt_stats_print is enabled.
*
* Synchronization: tcache_ql_mtx.
*/
ql_head(tcache_slow_t) tcache_ql;
ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql;
malloc_mutex_t tcache_ql_mtx;
/*
* Represents a dss_prec_t, but atomically.
*
* Synchronization: atomic.
*/
atomic_u_t dss_prec;
/*
* Extant large allocations.
*
* Synchronization: large_mtx.
*/
edata_list_active_t large;
/* Synchronizes all large allocation/update/deallocation. */
malloc_mutex_t large_mtx;
/* The page-level allocator shard this arena uses. */
pa_shard_t pa_shard;
/*
* A cached copy of base->ind. This can get accessed on hot paths;
* looking it up in base requires an extra pointer hop / cache miss.
*/
unsigned ind;
/*
* Base allocator, from which arena metadata are allocated.
*
* Synchronization: internal.
*/
base_t *base;
/* Used to determine uptime. Read-only after initialization. */
nstime_t create_time;
/* The name of the arena. */
char name[ARENA_NAME_LEN];
/*
* The arena is allocated alongside its bins; really this is a
* dynamically sized array determined by the binshard settings.
* Enforcing cacheline-alignment to minimize the number of cachelines
* touched on the hot paths.
*/
JEMALLOC_WARN_ON_USAGE(
"Do not use this field directly. "
"Use `arena_get_bin` instead.")
JEMALLOC_ALIGNED(CACHELINE)
bin_t all_bins[0];
};
#endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_H */

60
include/jemalloc/internal/arena_types.h Normal file
View File

@@ -0,0 +1,60 @@
#ifndef JEMALLOC_INTERNAL_ARENA_TYPES_H
#define JEMALLOC_INTERNAL_ARENA_TYPES_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/sc.h"
/* Default decay times in milliseconds. */
#define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000)
#define MUZZY_DECAY_MS_DEFAULT (0)
/* Number of event ticks between time checks. */
#define ARENA_DECAY_NTICKS_PER_UPDATE 1000
/* Maximum length of the arena name. */
#define ARENA_NAME_LEN 32
typedef struct arena_s arena_t;
typedef enum {
percpu_arena_mode_names_base = 0, /* Used for options processing. */
/*
* *_uninit are used only during bootstrapping, and must correspond
* to initialized variant plus percpu_arena_mode_enabled_base.
*/
percpu_arena_uninit = 0,
per_phycpu_arena_uninit = 1,
/* All non-disabled modes must come after percpu_arena_disabled. */
percpu_arena_disabled = 2,
percpu_arena_mode_names_limit = 3, /* Used for options processing. */
percpu_arena_mode_enabled_base = 3,
percpu_arena = 3,
per_phycpu_arena = 4 /* Hyper threads share arena. */
} percpu_arena_mode_t;
#define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base)
#define PERCPU_ARENA_DEFAULT percpu_arena_disabled
/*
* When allocation_size >= oversize_threshold, use the dedicated huge arena
* (unless have explicitly spicified arena index). 0 disables the feature.
*/
#define OVERSIZE_THRESHOLD_DEFAULT (8 << 20)
struct arena_config_s {
/* extent hooks to be used for the arena */
extent_hooks_t *extent_hooks;
/*
* Use extent hooks for metadata (base) allocations when true.
*/
bool metadata_use_hooks;
};
typedef struct arena_config_s arena_config_t;
extern const arena_config_t arena_config_default;
#endif /* JEMALLOC_INTERNAL_ARENA_TYPES_H */

48
include/jemalloc/internal/arenas_management.h
View File

@@ -1,48 +0,0 @@
#ifndef JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H
#define JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/tsd_types.h"
/*
* Arenas that are used to service external requests. Not all elements of the
* arenas array are necessarily used; arenas are created lazily as needed.
*
* arenas[0..narenas_auto) are used for automatic multiplexing of threads and
* arenas. arenas[narenas_auto..narenas_total) are only used if the application
* takes some action to create them and allocate from them.
*
* Points to an arena_t.
*/
extern atomic_p_t arenas[];
/* Number of arenas used for automatic multiplexing of threads and arenas. */
extern unsigned narenas_auto;
/* Base index for manual arenas. */
extern unsigned manual_arena_base;
void narenas_total_set(unsigned narenas);
void narenas_total_inc(void);
unsigned narenas_total_get(void);
void narenas_auto_set(unsigned n);
void manual_arena_base_set(unsigned base);
void *a0malloc(size_t size);
void a0dalloc(void *ptr);
void *a0ialloc(size_t size, bool zero, bool is_internal);
void a0idalloc(void *ptr, bool is_internal);
void arena_set(unsigned ind, arena_t *arena);
arena_t *arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
void arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena);
void iarena_cleanup(tsd_t *tsd);
void arena_cleanup(tsd_t *tsd);
bool arenas_management_boot(void);
void arenas_management_prefork(tsdn_t *tsdn);
void arenas_management_postfork_parent(tsdn_t *tsdn);
void arenas_management_postfork_child(tsdn_t *tsdn);
#endif /* JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H */

38
include/jemalloc/internal/background_thread_externs.h Normal file
View File

@@ -0,0 +1,38 @@
#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H
#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/background_thread_structs.h"
#include "jemalloc/internal/base.h"
#include "jemalloc/internal/mutex.h"
extern bool opt_background_thread;
extern size_t opt_max_background_threads;
extern malloc_mutex_t background_thread_lock;
extern atomic_b_t background_thread_enabled_state;
extern size_t n_background_threads;
extern size_t max_background_threads;
extern background_thread_info_t *background_thread_info;
bool background_thread_create(tsd_t *tsd, unsigned arena_ind);
bool background_threads_enable(tsd_t *tsd);
bool background_threads_disable(tsd_t *tsd);
bool background_thread_is_started(background_thread_info_t *info);
void background_thread_wakeup_early(
background_thread_info_t *info, nstime_t *remaining_sleep);
void background_thread_prefork0(tsdn_t *tsdn);
void background_thread_prefork1(tsdn_t *tsdn);
void background_thread_postfork_parent(tsdn_t *tsdn);
void background_thread_postfork_child(tsdn_t *tsdn);
bool background_thread_stats_read(
tsdn_t *tsdn, background_thread_stats_t *stats);
void background_thread_ctl_init(tsdn_t *tsdn);
#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
extern int pthread_create_wrapper(pthread_t *__restrict, const pthread_attr_t *,
void *(*)(void *), void *__restrict);
#endif
bool background_thread_boot0(void);
bool background_thread_boot1(tsdn_t *tsdn, base_t *base);
#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H */

4
include/jemalloc/internal/background_thread_inlines.h
View File

@@ -2,9 +2,9 @@
#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_INLINES_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena_inlines.h"
#include "jemalloc/internal/arena_inlines_a.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/background_thread.h"
#include "jemalloc/internal/background_thread_externs.h"
JEMALLOC_ALWAYS_INLINE bool
background_thread_enabled(void) {

38
include/jemalloc/internal/background_thread.h → include/jemalloc/internal/background_thread_structs.h
View File

@@ -1,10 +1,11 @@
#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_H
#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_H
#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H
#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/base.h"
#include "jemalloc/internal/mutex.h"
/* This file really combines "structs" and "types", but only transitionally. */
#if defined(JEMALLOC_BACKGROUND_THREAD) || defined(JEMALLOC_LAZY_LOCK)
# define JEMALLOC_PTHREAD_CREATE_WRAPPER
#endif
@@ -65,33 +66,4 @@ struct background_thread_stats_s {
};
typedef struct background_thread_stats_s background_thread_stats_t;
extern bool opt_background_thread;
extern size_t opt_max_background_threads;
extern malloc_mutex_t background_thread_lock;
extern atomic_b_t background_thread_enabled_state;
extern size_t n_background_threads;
extern size_t max_background_threads;
extern background_thread_info_t *background_thread_info;
bool background_thread_create(tsd_t *tsd, unsigned arena_ind);
bool background_threads_enable(tsd_t *tsd);
bool background_threads_disable(tsd_t *tsd);
bool background_thread_is_started(background_thread_info_t *info);
void background_thread_wakeup_early(
background_thread_info_t *info, nstime_t *remaining_sleep);
void background_thread_prefork0(tsdn_t *tsdn);
void background_thread_prefork1(tsdn_t *tsdn);
void background_thread_postfork_parent(tsdn_t *tsdn);
void background_thread_postfork_child(tsdn_t *tsdn);
bool background_thread_stats_read(
tsdn_t *tsdn, background_thread_stats_t *stats);
void background_thread_ctl_init(tsdn_t *tsdn);
#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
extern int pthread_create_wrapper(pthread_t *__restrict, const pthread_attr_t *,
void *(*)(void *), void *__restrict);
#endif
bool background_thread_boot0(void);
bool background_thread_boot1(tsdn_t *tsdn, base_t *base);
#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_H */
#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H */

35
include/jemalloc/internal/bin.h
View File

@@ -4,17 +4,11 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/bin_info.h"
#include "jemalloc/internal/bin_stats.h"
#include "jemalloc/internal/div.h"
#include "jemalloc/internal/bin_types.h"
#include "jemalloc/internal/edata.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/sc.h"
/* Forward decl; only used as a pointer type in bin_choose. */
typedef struct arena_s arena_t;
#define BIN_SHARDS_MAX (1 << EDATA_BITS_BINSHARD_WIDTH)
#define N_BIN_SHARDS_DEFAULT 1
/*
* A bin contains a set of extents that are currently being used for slab
* allocations.
@@ -101,18 +95,27 @@ bool bin_refill_slabcur_no_fresh_slab(tsdn_t *tsdn, bool is_auto,
void *bin_malloc_no_fresh_slab(tsdn_t *tsdn, bool is_auto, bin_t *bin,
szind_t binind);
/* Slab queries. */
void *bin_current_slab_addr(tsdn_t *tsdn, bin_t *bin);
/* Bin selection. */
bin_t *bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
unsigned *binshard_p);
typedef struct bin_dalloc_locked_info_s bin_dalloc_locked_info_t;
struct bin_dalloc_locked_info_s {
div_info_t div_info;
uint32_t nregs;
uint64_t ndalloc;
};
/* Stats. */
static inline void
bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) {
malloc_mutex_lock(tsdn, &bin->lock);
malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock);
bin_stats_t *stats = &dst_bin_stats->stats_data;
stats->nmalloc += bin->stats.nmalloc;
stats->ndalloc += bin->stats.ndalloc;
stats->nrequests += bin->stats.nrequests;
stats->curregs += bin->stats.curregs;
stats->nfills += bin->stats.nfills;
stats->nflushes += bin->stats.nflushes;
stats->nslabs += bin->stats.nslabs;
stats->reslabs += bin->stats.reslabs;
stats->curslabs += bin->stats.curslabs;
stats->nonfull_slabs += bin->stats.nonfull_slabs;
malloc_mutex_unlock(tsdn, &bin->lock);
}
#endif /* JEMALLOC_INTERNAL_BIN_H */

55
include/jemalloc/internal/bin_inlines.h
View File

@@ -7,38 +7,25 @@
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/div.h"
#include "jemalloc/internal/edata.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/sc.h"
/* Stats. */
static inline void
bin_stats_nrequests_add(tsdn_t *tsdn, bin_t *bin, uint64_t n) {
malloc_mutex_lock(tsdn, &bin->lock);
bin->stats.nrequests += n;
malloc_mutex_unlock(tsdn, &bin->lock);
}
static inline void
bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) {
malloc_mutex_lock(tsdn, &bin->lock);
malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock);
bin_stats_t *stats = &dst_bin_stats->stats_data;
stats->nmalloc += bin->stats.nmalloc;
stats->ndalloc += bin->stats.ndalloc;
stats->nrequests += bin->stats.nrequests;
stats->curregs += bin->stats.curregs;
stats->nfills += bin->stats.nfills;
stats->nflushes += bin->stats.nflushes;
stats->nslabs += bin->stats.nslabs;
stats->reslabs += bin->stats.reslabs;
stats->curslabs += bin->stats.curslabs;
stats->nonfull_slabs += bin->stats.nonfull_slabs;
malloc_mutex_unlock(tsdn, &bin->lock);
}
/*
* The dalloc bin info contains just the information that the common paths need
* during tcache flushes. By force-inlining these paths, and using local copies
* of data (so that the compiler knows it's constant), we avoid a whole bunch of
* redundant loads and stores by leaving this information in registers.
*/
typedef struct bin_dalloc_locked_info_s bin_dalloc_locked_info_t;
struct bin_dalloc_locked_info_s {
div_info_t div_info;
uint32_t nregs;
uint64_t ndalloc;
};
/* Find the region index of a pointer within a slab. */
JEMALLOC_ALWAYS_INLINE size_t
bin_slab_regind_impl(const div_info_t *div_info, szind_t binind,
const edata_t *slab, const void *ptr) {
bin_slab_regind_impl(
div_info_t *div_info, szind_t binind, edata_t *slab, const void *ptr) {
size_t diff, regind;
/* Freeing a pointer outside the slab can cause assertion failure. */
@@ -58,13 +45,21 @@ bin_slab_regind_impl(const div_info_t *div_info, szind_t binind,
}
JEMALLOC_ALWAYS_INLINE size_t
bin_slab_regind(const bin_dalloc_locked_info_t *info, szind_t binind,
const edata_t *slab, const void *ptr) {
bin_slab_regind(bin_dalloc_locked_info_t *info, szind_t binind,
edata_t *slab, const void *ptr) {
size_t regind = bin_slab_regind_impl(
&info->div_info, binind, slab, ptr);
return regind;
}
JEMALLOC_ALWAYS_INLINE void
bin_dalloc_locked_begin(
bin_dalloc_locked_info_t *info, szind_t binind) {
info->div_info = arena_binind_div_info[binind];
info->nregs = bin_infos[binind].nregs;
info->ndalloc = 0;
}
/*
* Does the deallocation work associated with freeing a single pointer (a
* "step") in between a bin_dalloc_locked begin and end call.

21
include/jemalloc/internal/bin_types.h Normal file
View File

@@ -0,0 +1,21 @@
#ifndef JEMALLOC_INTERNAL_BIN_TYPES_H
#define JEMALLOC_INTERNAL_BIN_TYPES_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/sc.h"
#define BIN_SHARDS_MAX (1 << EDATA_BITS_BINSHARD_WIDTH)
#define N_BIN_SHARDS_DEFAULT 1
/* Used in TSD static initializer only. Real init in arena_bind(). */
#define TSD_BINSHARDS_ZERO_INITIALIZER \
{ \
{ UINT8_MAX } \
}
typedef struct tsd_binshards_s tsd_binshards_t;
struct tsd_binshards_s {
uint8_t binshard[SC_NBINS];
};
#endif /* JEMALLOC_INTERNAL_BIN_TYPES_H */

4
include/jemalloc/internal/bit_util.h
View File

@@ -195,7 +195,7 @@ fls_u(unsigned x) {
/* \
* Algorithm from an old AMD optimization reference manual. \
* We're putting a little bit more work than you might expect \
* into the no-intrinsic case, since we only support the \
* into the no-instrinsic case, since we only support the \
* GCC intrinsics spelling of popcount (for now). Detecting \
* whether or not the popcount builtin is actually useable in \
* MSVC is nontrivial. \
@@ -212,7 +212,7 @@ fls_u(unsigned x) {
* That costs an extra mask relative to this, though. \
*/ \
x = x - ((x >> 1) & (0x55U * bmul)); \
/* Replace each 4 bits with their sideways sum. 0x3 = 0b0011. */\
/* Replace each 4 bits with their sideays sum. 0x3 = 0b0011. */\
x = (x & (bmul * 0x33U)) + ((x >> 2) & (bmul * 0x33U)); \
/* \
* Replace each 8 bits with their sideways sum. Note that we \

4
include/jemalloc/internal/bitmap.h
View File

@@ -181,7 +181,7 @@ void bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill);
size_t bitmap_size(const bitmap_info_t *binfo);
static inline bool
bitmap_full(const bitmap_t *bitmap, const bitmap_info_t *binfo) {
bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) {
#ifdef BITMAP_USE_TREE
size_t rgoff = binfo->levels[binfo->nlevels].group_offset - 1;
bitmap_t rg = bitmap[rgoff];
@@ -200,7 +200,7 @@ bitmap_full(const bitmap_t *bitmap, const bitmap_info_t *binfo) {
}
static inline bool
bitmap_get(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
size_t goff;
bitmap_t g;

16
include/jemalloc/internal/cache_bin.h
View File

@@ -442,6 +442,19 @@ cache_bin_alloc(cache_bin_t *bin, bool *success) {
return cache_bin_alloc_impl(bin, success, true);
}
JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
cache_bin_alloc_batch(cache_bin_t *bin, size_t num, void **out) {
cache_bin_sz_t n = cache_bin_ncached_get_internal(bin);
if (n > num) {
n = (cache_bin_sz_t)num;
}
memcpy(out, bin->stack_head, n * sizeof(void *));
bin->stack_head += n;
cache_bin_low_water_adjust(bin);
return n;
}
JEMALLOC_ALWAYS_INLINE bool
cache_bin_full(cache_bin_t *bin) {
return (
@@ -523,13 +536,14 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) {
/* Get the number of stashed pointers. */
JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
cache_bin_nstashed_get_internal(cache_bin_t *bin) {
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin);
cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(
bin);
cache_bin_sz_t n = cache_bin_diff(
bin, low_bits_low_bound, bin->low_bits_full)
/ sizeof(void *);
assert(n <= cache_bin_ncached_max_get(bin));
assert(n <= ncached_max);
if (config_debug && n != 0) {
/* Below are for assertions only. */
void **low_bound = cache_bin_low_bound_get(bin);

2
include/jemalloc/internal/conf.h
View File

@@ -7,9 +7,9 @@ void malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
char readlink_buf[PATH_MAX + 1]);
void malloc_abort_invalid_conf(void);
#ifdef JEMALLOC_JET
extern bool had_conf_error;
#ifdef JEMALLOC_JET
bool conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
char const **v_p, size_t *vlen_p);
void conf_error(

3
include/jemalloc/internal/ctl.h
View File

@@ -3,7 +3,7 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena_stats.h"
#include "jemalloc/internal/background_thread.h"
#include "jemalloc/internal/background_thread_structs.h"
#include "jemalloc/internal/bin_stats.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/malloc_io.h"
@@ -63,7 +63,6 @@ typedef struct ctl_stats_s {
size_t resident;
size_t mapped;
size_t retained;
size_t pinned;
background_thread_stats_t background_thread;
mutex_prof_data_t mutex_prof_data[mutex_prof_num_global_mutexes];

2
include/jemalloc/internal/div.h
View File

@@ -24,7 +24,7 @@ struct div_info_s {
void div_init(div_info_t *div_info, size_t divisor);
static inline size_t
div_compute(const div_info_t *div_info, size_t n) {
div_compute(div_info_t *div_info, size_t n) {
assert(n <= (uint32_t)-1);
/*
* This generates, e.g. mov; imul; shr on x86-64. On a 32-bit machine,

8
include/jemalloc/internal/ecache.h
View File

@@ -23,27 +23,27 @@ struct ecache_s {
};
static inline size_t
ecache_npages_get(const ecache_t *ecache) {
ecache_npages_get(ecache_t *ecache) {
return eset_npages_get(&ecache->eset)
+ eset_npages_get(&ecache->guarded_eset);
}
/* Get the number of extents in the given page size index. */
static inline size_t
ecache_nextents_get(const ecache_t *ecache, pszind_t ind) {
ecache_nextents_get(ecache_t *ecache, pszind_t ind) {
return eset_nextents_get(&ecache->eset, ind)
+ eset_nextents_get(&ecache->guarded_eset, ind);
}
/* Get the sum total bytes of the extents in the given page size index. */
static inline size_t
ecache_nbytes_get(const ecache_t *ecache, pszind_t ind) {
ecache_nbytes_get(ecache_t *ecache, pszind_t ind) {
return eset_nbytes_get(&ecache->eset, ind)
+ eset_nbytes_get(&ecache->guarded_eset, ind);
}
static inline unsigned
ecache_ind_get(const ecache_t *ecache) {
ecache_ind_get(ecache_t *ecache) {
return ecache->ind;
}

65
include/jemalloc/internal/edata.h
View File

@@ -8,16 +8,13 @@
#include "jemalloc/internal/hpdata.h"
#include "jemalloc/internal/nstime.h"
#include "jemalloc/internal/ph.h"
#include "jemalloc/internal/prof_types.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/slab_data.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/typed_list.h"
/* Opaque to edata; only stored as pointers in e_prof_info_t. */
typedef struct prof_tctx_s prof_tctx_t;
typedef struct prof_recent_s prof_recent_t;
/*
* sizeof(edata_t) is 128 bytes on 64-bit architectures. Ensure the alignment
* to free up the low bits in the rtree leaf.
@@ -37,10 +34,9 @@ enum extent_state_e {
extent_state_dirty = 1,
extent_state_muzzy = 2,
extent_state_retained = 3,
extent_state_pinned = 4,
extent_state_transition = 5, /* States below are intermediate. */
extent_state_merging = 6,
extent_state_max = 6 /* Sanity checking only. */
extent_state_transition = 4, /* States below are intermediate. */
extent_state_merging = 5,
extent_state_max = 5 /* Sanity checking only. */
};
typedef enum extent_state_e extent_state_t;
@@ -51,8 +47,8 @@ enum extent_head_state_e {
typedef enum extent_head_state_e extent_head_state_t;
/*
* Which page allocator implementation (PAC or HPA) owns the given extent?
* Used by PA to route expand/shrink/dalloc to the correct implementation.
* Which implementation of the page allocator interface, (PAI, defined in
* pai.h) owns the given extent?
*/
enum extent_pai_e { EXTENT_PAI_PAC = 0, EXTENT_PAI_HPA = 1 };
typedef enum extent_pai_e extent_pai_t;
@@ -98,8 +94,8 @@ struct edata_cmp_summary_s {
/* Extent (span of pages). Use accessor functions for e_* fields. */
typedef struct edata_s edata_t;
ph_structs(edata_avail, edata_t, ESET_ENUMERATE_MAX_NUM)
ph_structs(edata_heap, edata_t, ESET_ENUMERATE_MAX_NUM)
ph_structs(edata_avail, edata_t, ESET_ENUMERATE_MAX_NUM);
ph_structs(edata_heap, edata_t, ESET_ENUMERATE_MAX_NUM);
struct edata_s {
/*
* Bitfield containing several fields:
@@ -114,10 +110,8 @@ struct edata_s {
* i: szind
* f: nfree
* s: bin_shard
* h: is_head
* n: pinned
*
* 00000000 ... 0nhsssss ssffffff ffffiiii iiiitttg zpcbaaaa aaaaaaaa
* 00000000 ... 0000ssss ssffffff ffffiiii iiiitttg zpcbaaaa aaaaaaaa
*
* arena_ind: Arena from which this extent came, or all 1 bits if
* unassociated.
@@ -151,10 +145,6 @@ struct edata_s {
* nfree: Number of free regions in slab.
*
* bin_shard: the shard of the bin from which this extent came.
*
* is_head: see comments in ehooks_default_merge_impl().
*
* pinned: true if the alloc hook signaled non-reclaimable backing.
*/
uint64_t e_bits;
#define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT) \
@@ -220,16 +210,6 @@ struct edata_s {
#define EDATA_BITS_IS_HEAD_MASK \
MASK(EDATA_BITS_IS_HEAD_WIDTH, EDATA_BITS_IS_HEAD_SHIFT)
#define EDATA_BITS_PINNED_WIDTH 1
#define EDATA_BITS_PINNED_SHIFT \
(EDATA_BITS_IS_HEAD_WIDTH + EDATA_BITS_IS_HEAD_SHIFT)
#define EDATA_BITS_PINNED_MASK \
MASK(EDATA_BITS_PINNED_WIDTH, EDATA_BITS_PINNED_SHIFT)
#if (EDATA_BITS_PINNED_SHIFT + EDATA_BITS_PINNED_WIDTH > 64)
#error "edata_t e_bits overflow"
#endif
/* Pointer to the extent that this structure is responsible for. */
void *e_addr;
@@ -558,29 +538,6 @@ edata_ps_set(edata_t *edata, hpdata_t *ps) {
edata->e_ps = ps;
}
static inline bool
edata_pinned_get(const edata_t *edata) {
return (bool)((edata->e_bits & EDATA_BITS_PINNED_MASK)
>> EDATA_BITS_PINNED_SHIFT);
}
static inline void
edata_pinned_set(edata_t *edata, bool pinned) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_PINNED_MASK)
| ((uint64_t)pinned << EDATA_BITS_PINNED_SHIFT);
}
static inline void
edata_hook_flags_init(edata_t *edata, unsigned alloc_flags) {
edata_pinned_set(edata,
(alloc_flags & EXTENT_ALLOC_FLAG_PINNED) != 0);
}
static inline unsigned
edata_alloc_flags_get(const edata_t *edata) {
return edata_pinned_get(edata) ? EXTENT_ALLOC_FLAG_PINNED : 0;
}
static inline void
edata_szind_set(edata_t *edata, szind_t szind) {
assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */
@@ -688,7 +645,7 @@ edata_prof_recent_alloc_set_dont_call_directly(
}
static inline bool
edata_is_head_get(const edata_t *edata) {
edata_is_head_get(edata_t *edata) {
return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK)
>> EDATA_BITS_IS_HEAD_SHIFT);
}
@@ -729,7 +686,6 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
edata_committed_set(edata, committed);
edata_pai_set(edata, pai);
edata_is_head_set(edata, is_head == EXTENT_IS_HEAD);
edata_hook_flags_init(edata, 0);
if (config_prof) {
edata_prof_tctx_set(edata, NULL);
}
@@ -755,7 +711,6 @@ edata_binit(
* wasting a state bit to encode this fact.
*/
edata_pai_set(edata, EXTENT_PAI_PAC);
edata_hook_flags_init(edata, 0);
}
static inline int

26
include/jemalloc/internal/ehooks.h
View File

@@ -110,12 +110,12 @@ ehooks_set_extent_hooks_ptr(ehooks_t *ehooks, extent_hooks_t *extent_hooks) {
}
static inline extent_hooks_t *
ehooks_get_extent_hooks_ptr(const ehooks_t *ehooks) {
ehooks_get_extent_hooks_ptr(ehooks_t *ehooks) {
return (extent_hooks_t *)atomic_load_p(&ehooks->ptr, ATOMIC_ACQUIRE);
}
static inline bool
ehooks_are_default(const ehooks_t *ehooks) {
ehooks_are_default(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)
== &ehooks_default_extent_hooks;
}
@@ -126,7 +126,7 @@ ehooks_are_default(const ehooks_t *ehooks) {
* include some checks for such cases.
*/
static inline bool
ehooks_dalloc_will_fail(const ehooks_t *ehooks) {
ehooks_dalloc_will_fail(ehooks_t *ehooks) {
if (ehooks_are_default(ehooks)) {
return opt_retain;
} else {
@@ -135,17 +135,17 @@ ehooks_dalloc_will_fail(const ehooks_t *ehooks) {
}
static inline bool
ehooks_split_will_fail(const ehooks_t *ehooks) {
ehooks_split_will_fail(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->split == NULL;
}
static inline bool
ehooks_merge_will_fail(const ehooks_t *ehooks) {
ehooks_merge_will_fail(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL;
}
static inline bool
ehooks_guard_will_fail(const ehooks_t *ehooks) {
ehooks_guard_will_fail(ehooks_t *ehooks) {
/*
* Before the guard hooks are officially introduced, limit the use to
* the default hooks only.
@@ -191,7 +191,7 @@ ehooks_debug_zero_check(void *addr, size_t size) {
static inline void *
ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned *alloc_flags) {
size_t alignment, bool *zero, bool *commit) {
bool orig_zero = *zero;
void *ret;
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
@@ -204,18 +204,6 @@ ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
alignment, zero, commit, ehooks_ind_get(ehooks));
ehooks_post_reentrancy(tsdn);
}
#if LG_PAGE < 8
# error "Extent alloc flags require page size of at least 256"
#endif
if (ret != NULL) {
*alloc_flags = (unsigned)((uintptr_t)ret
& EXTENT_ALLOC_FLAG_MASK);
ret = (void *)((byte_t *)ret - *alloc_flags);
/* Pinned hooks must also set *commit; pinned bypasses commit/decommit. */
assert(!(*alloc_flags & EXTENT_ALLOC_FLAG_PINNED) || *commit);
} else {
*alloc_flags = 0;
}
assert(new_addr == NULL || ret == NULL || new_addr == ret);
assert(!orig_zero || *zero);
if (*zero && ret != NULL) {

1
include/jemalloc/internal/emap.h
View File

@@ -211,7 +211,6 @@ extent_assert_can_coalesce(const edata_t *inner, const edata_t *outer) {
assert(edata_state_get(inner) == extent_state_active);
assert(edata_state_get(outer) == extent_state_merging);
assert(!edata_guarded_get(inner) && !edata_guarded_get(outer));
assert(edata_pinned_get(inner) == edata_pinned_get(outer));
assert(edata_base_get(inner) == edata_past_get(outer)
|| edata_base_get(outer) == edata_past_get(inner));
}

8
include/jemalloc/internal/eset.h
View File

@@ -60,11 +60,11 @@ struct eset_s {
void eset_init(eset_t *eset, extent_state_t state);
size_t eset_npages_get(const eset_t *eset);
size_t eset_npages_get(eset_t *eset);
/* Get the number of extents in the given page size index. */
size_t eset_nextents_get(const eset_t *eset, pszind_t ind);
size_t eset_nextents_get(eset_t *eset, pszind_t ind);
/* Get the sum total bytes of the extents in the given page size index. */
size_t eset_nbytes_get(const eset_t *eset, pszind_t ind);
size_t eset_nbytes_get(eset_t *eset, pszind_t ind);
void eset_insert(eset_t *eset, edata_t *edata);
void eset_remove(eset_t *eset, edata_t *edata);
@@ -73,6 +73,6 @@ void eset_remove(eset_t *eset, edata_t *edata);
* null if no such item could be found.
*/
edata_t *eset_fit(eset_t *eset, size_t esize, size_t alignment, bool exact_only,
unsigned lg_max_fit, bool prefer_small);
unsigned lg_max_fit);
#endif /* JEMALLOC_INTERNAL_ESET_H */

6
include/jemalloc/internal/extent.h
View File

@@ -91,7 +91,7 @@ extent_neighbor_head_state_mergeable(
}
JEMALLOC_ALWAYS_INLINE bool
extent_can_acquire_neighbor(const edata_t *edata, rtree_contents_t contents,
extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
extent_pai_t pai, extent_state_t expected_state, bool forward,
bool expanding) {
edata_t *neighbor = contents.edata;
@@ -120,10 +120,6 @@ extent_can_acquire_neighbor(const edata_t *edata, rtree_contents_t contents,
*/
return false;
}
/* Do not merge pinned and non-pinned extents. */
if (edata_pinned_get(edata) != edata_pinned_get(neighbor)) {
return false;
}
} else {
if (neighbor_state == extent_state_active) {
return false;

9
include/jemalloc/internal/extent_dss.h
View File

@@ -2,11 +2,9 @@
#define JEMALLOC_INTERNAL_EXTENT_DSS_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena_types.h"
#include "jemalloc/internal/tsd_types.h"
/* Forward decl; arena.h includes us, so we can't include arena.h back. */
typedef struct arena_s arena_t;
typedef enum {
dss_prec_disabled = 0,
dss_prec_primary = 1,
@@ -29,9 +27,4 @@ bool extent_in_dss(void *addr);
bool extent_dss_mergeable(void *addr_a, void *addr_b);
void extent_dss_boot(void);
#ifdef JEMALLOC_JET
typedef void *(*extent_dss_sbrk_hook_t)(intptr_t);
extern extent_dss_sbrk_hook_t extent_dss_sbrk_hook;
#endif
#endif /* JEMALLOC_INTERNAL_EXTENT_DSS_H */

56
include/jemalloc/internal/fb.h
View File

@@ -25,7 +25,7 @@ fb_init(fb_group_t *fb, size_t nbits) {
}
static inline bool
fb_empty(const fb_group_t *fb, size_t nbits) {
fb_empty(fb_group_t *fb, size_t nbits) {
size_t ngroups = FB_NGROUPS(nbits);
for (size_t i = 0; i < ngroups; i++) {
if (fb[i] != 0) {
@@ -36,7 +36,7 @@ fb_empty(const fb_group_t *fb, size_t nbits) {
}
static inline bool
fb_full(const fb_group_t *fb, size_t nbits) {
fb_full(fb_group_t *fb, size_t nbits) {
size_t ngroups = FB_NGROUPS(nbits);
size_t trailing_bits = nbits % FB_GROUP_BITS;
size_t limit = (trailing_bits == 0 ? ngroups : ngroups - 1);
@@ -52,7 +52,7 @@ fb_full(const fb_group_t *fb, size_t nbits) {
}
static inline bool
fb_get(const fb_group_t *fb, size_t nbits, size_t bit) {
fb_get(fb_group_t *fb, size_t nbits, size_t bit) {
assert(bit < nbits);
size_t group_ind = bit / FB_GROUP_BITS;
size_t bit_ind = bit % FB_GROUP_BITS;
@@ -156,21 +156,15 @@ fb_scount_visitor(void *ctx, fb_group_t *fb, fb_group_t mask) {
/* Finds the number of set bit in the of length cnt starting at start. */
JEMALLOC_ALWAYS_INLINE size_t
fb_scount(const fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
fb_scount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
size_t scount = 0;
/*
* fb_visit_impl is shared with mutating visitors (e.g. fb_set_range),
* so it takes a non-const fb. fb_scount_visitor only reads, so the
* cast is safe.
*/
fb_visit_impl((fb_group_t *)fb, nbits, &fb_scount_visitor, &scount,
start, cnt);
fb_visit_impl(fb, nbits, &fb_scount_visitor, &scount, start, cnt);
return scount;
}
/* Finds the number of unset bit in the of length cnt starting at start. */
JEMALLOC_ALWAYS_INLINE size_t
fb_ucount(const fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
fb_ucount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
size_t scount = fb_scount(fb, nbits, start, cnt);
return cnt - scount;
}
@@ -182,8 +176,8 @@ fb_ucount(const fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
* Returns the number of bits in the bitmap if no such bit exists.
*/
JEMALLOC_ALWAYS_INLINE ssize_t
fb_find_impl(const fb_group_t *fb, size_t nbits, size_t start, bool val,
bool forward) {
fb_find_impl(
fb_group_t *fb, size_t nbits, size_t start, bool val, bool forward) {
assert(start < nbits);
size_t ngroups = FB_NGROUPS(nbits);
ssize_t group_ind = start / FB_GROUP_BITS;
@@ -232,14 +226,14 @@ fb_find_impl(const fb_group_t *fb, size_t nbits, size_t start, bool val,
* number of bits in the bitmap if no such bit exists.
*/
static inline size_t
fb_ffu(const fb_group_t *fb, size_t nbits, size_t min_bit) {
fb_ffu(fb_group_t *fb, size_t nbits, size_t min_bit) {
return (size_t)fb_find_impl(fb, nbits, min_bit, /* val */ false,
/* forward */ true);
}
/* The same, but looks for an unset bit. */
static inline size_t
fb_ffs(const fb_group_t *fb, size_t nbits, size_t min_bit) {
fb_ffs(fb_group_t *fb, size_t nbits, size_t min_bit) {
return (size_t)fb_find_impl(fb, nbits, min_bit, /* val */ true,
/* forward */ true);
}
@@ -249,21 +243,21 @@ fb_ffs(const fb_group_t *fb, size_t nbits, size_t min_bit) {
* no such bit exists.
*/
static inline ssize_t
fb_flu(const fb_group_t *fb, size_t nbits, size_t max_bit) {
fb_flu(fb_group_t *fb, size_t nbits, size_t max_bit) {
return fb_find_impl(fb, nbits, max_bit, /* val */ false,
/* forward */ false);
}
static inline ssize_t
fb_fls(const fb_group_t *fb, size_t nbits, size_t max_bit) {
fb_fls(fb_group_t *fb, size_t nbits, size_t max_bit) {
return fb_find_impl(fb, nbits, max_bit, /* val */ true,
/* forward */ false);
}
/* Returns whether or not we found a range. */
JEMALLOC_ALWAYS_INLINE bool
fb_iter_range_impl(const fb_group_t *fb, size_t nbits, size_t start,
size_t *r_begin, size_t *r_len, bool val, bool forward) {
fb_iter_range_impl(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
size_t *r_len, bool val, bool forward) {
assert(start < nbits);
ssize_t next_range_begin = fb_find_impl(fb, nbits, start, val, forward);
if ((forward && next_range_begin == (ssize_t)nbits)
@@ -292,8 +286,8 @@ fb_iter_range_impl(const fb_group_t *fb, size_t nbits, size_t start,
* touching *r_begin or *r_end).
*/
static inline bool
fb_srange_iter(const fb_group_t *fb, size_t nbits, size_t start,
size_t *r_begin, size_t *r_len) {
fb_srange_iter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
size_t *r_len) {
return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
/* val */ true, /* forward */ true);
}
@@ -303,30 +297,30 @@ fb_srange_iter(const fb_group_t *fb, size_t nbits, size_t start,
* forwards. (The position returned is still the earliest bit in the range).
*/
static inline bool
fb_srange_riter(const fb_group_t *fb, size_t nbits, size_t start,
size_t *r_begin, size_t *r_len) {
fb_srange_riter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
size_t *r_len) {
return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
/* val */ true, /* forward */ false);
}
/* Similar to fb_srange_iter, but searches for unset bits. */
static inline bool
fb_urange_iter(const fb_group_t *fb, size_t nbits, size_t start,
size_t *r_begin, size_t *r_len) {
fb_urange_iter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
size_t *r_len) {
return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
/* val */ false, /* forward */ true);
}
/* Similar to fb_srange_riter, but searches for unset bits. */
static inline bool
fb_urange_riter(const fb_group_t *fb, size_t nbits, size_t start,
size_t *r_begin, size_t *r_len) {
fb_urange_riter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
size_t *r_len) {
return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
/* val */ false, /* forward */ false);
}
JEMALLOC_ALWAYS_INLINE size_t
fb_range_longest_impl(const fb_group_t *fb, size_t nbits, bool val) {
fb_range_longest_impl(fb_group_t *fb, size_t nbits, bool val) {
size_t begin = 0;
size_t longest_len = 0;
size_t len = 0;
@@ -342,12 +336,12 @@ fb_range_longest_impl(const fb_group_t *fb, size_t nbits, bool val) {
}
static inline size_t
fb_srange_longest(const fb_group_t *fb, size_t nbits) {
fb_srange_longest(fb_group_t *fb, size_t nbits) {
return fb_range_longest_impl(fb, nbits, /* val */ true);
}
static inline size_t
fb_urange_longest(const fb_group_t *fb, size_t nbits) {
fb_urange_longest(fb_group_t *fb, size_t nbits) {
return fb_range_longest_impl(fb, nbits, /* val */ false);
}

163
include/jemalloc/internal/hook.h Normal file
View File

@@ -0,0 +1,163 @@
#ifndef JEMALLOC_INTERNAL_HOOK_H
#define JEMALLOC_INTERNAL_HOOK_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/tsd.h"
/*
* This API is *extremely* experimental, and may get ripped out, changed in API-
* and ABI-incompatible ways, be insufficiently or incorrectly documented, etc.
*
* It allows hooking the stateful parts of the API to see changes as they
* happen.
*
* Allocation hooks are called after the allocation is done, free hooks are
* called before the free is done, and expand hooks are called after the
* allocation is expanded.
*
* For realloc and rallocx, if the expansion happens in place, the expansion
* hook is called. If it is moved, then the alloc hook is called on the new
* location, and then the free hook is called on the old location (i.e. both
* hooks are invoked in between the alloc and the dalloc).
*
* If we return NULL from OOM, then usize might not be trustworthy. Calling
* realloc(NULL, size) only calls the alloc hook, and calling realloc(ptr, 0)
* only calls the free hook. (Calling realloc(NULL, 0) is treated as malloc(0),
* and only calls the alloc hook).
*
* Reentrancy:
* Reentrancy is guarded against from within the hook implementation. If you
* call allocator functions from within a hook, the hooks will not be invoked
* again.
* Threading:
* The installation of a hook synchronizes with all its uses. If you can
* prove the installation of a hook happens-before a jemalloc entry point,
* then the hook will get invoked (unless there's a racing removal).
*
* Hook insertion appears to be atomic at a per-thread level (i.e. if a thread
* allocates and has the alloc hook invoked, then a subsequent free on the
* same thread will also have the free hook invoked).
*
* The *removal* of a hook does *not* block until all threads are done with
* the hook. Hook authors have to be resilient to this, and need some
* out-of-band mechanism for cleaning up any dynamically allocated memory
* associated with their hook.
* Ordering:
* Order of hook execution is unspecified, and may be different than insertion
* order.
*/
#define HOOK_MAX 4
enum hook_alloc_e {
hook_alloc_malloc,
hook_alloc_posix_memalign,
hook_alloc_aligned_alloc,
hook_alloc_calloc,
hook_alloc_memalign,
hook_alloc_valloc,
hook_alloc_pvalloc,
hook_alloc_mallocx,
/* The reallocating functions have both alloc and dalloc variants */
hook_alloc_realloc,
hook_alloc_rallocx,
};
/*
* We put the enum typedef after the enum, since this file may get included by
* jemalloc_cpp.cpp, and C++ disallows enum forward declarations.
*/
typedef enum hook_alloc_e hook_alloc_t;
enum hook_dalloc_e {
hook_dalloc_free,
hook_dalloc_dallocx,
hook_dalloc_sdallocx,
/*
* The dalloc halves of reallocation (not called if in-place expansion
* happens).
*/
hook_dalloc_realloc,
hook_dalloc_rallocx,
};
typedef enum hook_dalloc_e hook_dalloc_t;
enum hook_expand_e {
hook_expand_realloc,
hook_expand_rallocx,
hook_expand_xallocx,
};
typedef enum hook_expand_e hook_expand_t;
typedef void (*hook_alloc)(void *extra, hook_alloc_t type, void *result,
uintptr_t result_raw, uintptr_t args_raw[3]);
typedef void (*hook_dalloc)(
void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
typedef void (*hook_expand)(void *extra, hook_expand_t type, void *address,
size_t old_usize, size_t new_usize, uintptr_t result_raw,
uintptr_t args_raw[4]);
typedef struct hooks_s hooks_t;
struct hooks_s {
hook_alloc alloc_hook;
hook_dalloc dalloc_hook;
hook_expand expand_hook;
void *extra;
};
/*
* Begin implementation details; everything above this point might one day live
* in a public API. Everything below this point never will.
*/
/*
* The realloc pathways haven't gotten any refactoring love in a while, and it's
* fairly difficult to pass information from the entry point to the hooks. We
* put the informaiton the hooks will need into a struct to encapsulate
* everything.
*
* Much of these pathways are force-inlined, so that the compiler can avoid
* materializing this struct until we hit an extern arena function. For fairly
* goofy reasons, *many* of the realloc paths hit an extern arena function.
* These paths are cold enough that it doesn't matter; eventually, we should
* rewrite the realloc code to make the expand-in-place and the
* free-then-realloc paths more orthogonal, at which point we don't need to
* spread the hook logic all over the place.
*/
typedef struct hook_ralloc_args_s hook_ralloc_args_t;
struct hook_ralloc_args_s {
/* I.e. as opposed to rallocx. */
bool is_realloc;
/*
* The expand hook takes 4 arguments, even if only 3 are actually used;
* we add an extra one in case the user decides to memcpy without
* looking too closely at the hooked function.
*/
uintptr_t args[4];
};
/*
* Returns an opaque handle to be used when removing the hook. NULL means that
* we couldn't install the hook.
*/
bool hook_boot(void);
void *hook_install(tsdn_t *tsdn, hooks_t *to_install);
/* Uninstalls the hook with the handle previously returned from hook_install. */
void hook_remove(tsdn_t *tsdn, void *opaque);
/* Hooks */
void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
uintptr_t args_raw[3]);
void hook_invoke_dalloc(
hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);
#endif /* JEMALLOC_INTERNAL_HOOK_H */

75
include/jemalloc/internal/hpa.h
View File

@@ -10,6 +10,7 @@
#include "jemalloc/internal/hpa_hooks.h"
#include "jemalloc/internal/hpa_opts.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/pai.h"
#include "jemalloc/internal/psset.h"
#include "jemalloc/internal/sec.h"
@@ -50,60 +51,6 @@ struct hpa_shard_nonderived_stats_s {
* Guarded by mtx.
*/
uint64_t ndehugifies;
/*
* Distribution of the min number of extents we will try to allocate
* from a single hpa_alloc() call.
*
* Guarded by mtx.
*/
uint64_t hpa_alloc_min_extents[SEC_MAX_NALLOCS + 1];
/*
* Distribution of the max number of extents we will try to allocate
* from a single hpa_alloc() call.
*
* Guarded by mtx.
*/
uint64_t hpa_alloc_max_extents[SEC_MAX_NALLOCS + 1];
/*
* Distribution of the number of extents allocated for a single
* hpa_alloc() call and a single mtx lock acquisition.
*
* Guarded by mtx.
*/
uint64_t hpa_alloc_extents[SEC_MAX_NALLOCS + 1];
/*
* Distribution of the number of ps out of which we allocated extents
* for a single hpa_alloc() call and a single mtx lock acquisition.
*
* Guarded by mtx.
*/
uint64_t hpa_alloc_ps[SEC_MAX_NALLOCS + 1];
/*
* Distribution of the number of pages allocated from a single ps.
*
* Guarded by mtx.
*/
uint64_t hpa_alloc_pages_per_ps[SEC_MAX_NALLOCS + 1];
/*
* Distribution of the number of extents allocated from a single ps.
*
* Guarded by mtx.
*/
uint64_t hpa_alloc_extents_per_ps[SEC_MAX_NALLOCS + 1];
/*
* Distribution of the total elapsed time (ns) for allocating extents
* from a single ps.
*
* Guarded by mtx.
*/
uint64_t hpa_alloc_total_elapsed_ns_per_ps[SEC_MAX_NALLOCS + 1];
};
/* Completely derived; only used by CTL. */
@@ -116,6 +63,12 @@ struct hpa_shard_stats_s {
typedef struct hpa_shard_s hpa_shard_t;
struct hpa_shard_s {
/*
* pai must be the first member; we cast from a pointer to it to a
* pointer to the hpa_shard_t.
*/
pai_t pai;
/* The central allocator we get our hugepages from. */
hpa_central_t *central;
@@ -192,7 +145,7 @@ struct hpa_shard_s {
bool hpa_hugepage_size_exceeds_limit(void);
/*
* Whether or not the HPA can be used given the current configuration. This
* Whether or not the HPA can be used given the current configuration. This is
* is not necessarily a guarantee that it backs its allocations by hugepages,
* just that it can function properly given the system it's running on.
*/
@@ -201,18 +154,6 @@ bool hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind,
const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts);
edata_t *hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
size_t alignment, bool zero, bool guarded, bool frequent_reuse,
bool *deferred_work_generated);
bool hpa_expand(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
size_t old_size, size_t new_size, bool zero,
bool *deferred_work_generated);
bool hpa_shrink(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
size_t old_size, size_t new_size, bool *deferred_work_generated);
void hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
bool *deferred_work_generated);
uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard);
void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src);
void hpa_shard_stats_merge(
tsdn_t *tsdn, hpa_shard_t *shard, hpa_shard_stats_t *dst);

66
include/jemalloc/internal/hpdata.h
View File

@@ -27,7 +27,7 @@
*/
#define PSSET_ENUMERATE_MAX_NUM 32
typedef struct hpdata_s hpdata_t;
ph_structs(hpdata_age_heap, hpdata_t, PSSET_ENUMERATE_MAX_NUM)
ph_structs(hpdata_age_heap, hpdata_t, PSSET_ENUMERATE_MAX_NUM);
struct hpdata_s {
/*
* We likewise follow the edata convention of mangling names and forcing
@@ -136,7 +136,7 @@ TYPED_LIST(hpdata_empty_list, hpdata_t, ql_link_empty)
TYPED_LIST(hpdata_purge_list, hpdata_t, ql_link_purge)
TYPED_LIST(hpdata_hugify_list, hpdata_t, ql_link_hugify)
ph_proto(, hpdata_age_heap, hpdata_t)
ph_proto(, hpdata_age_heap, hpdata_t);
static inline void *
hpdata_addr_get(const hpdata_t *hpdata) {
@@ -209,7 +209,7 @@ hpdata_allow_hugify(hpdata_t *hpdata, nstime_t now) {
}
static inline nstime_t
hpdata_time_hugify_allowed(const hpdata_t *hpdata) {
hpdata_time_hugify_allowed(hpdata_t *hpdata) {
return hpdata->h_time_hugify_allowed;
}
@@ -305,7 +305,7 @@ hpdata_ndirty_get(const hpdata_t *hpdata) {
}
static inline size_t
hpdata_nretained_get(const hpdata_t *hpdata) {
hpdata_nretained_get(hpdata_t *hpdata) {
return HUGEPAGE_PAGES - hpdata->h_ntouched;
}
@@ -330,7 +330,7 @@ hpdata_purged_when_empty_and_huge_set(hpdata_t *hpdata, bool v) {
}
static inline void
hpdata_assert_empty(const hpdata_t *hpdata) {
hpdata_assert_empty(hpdata_t *hpdata) {
assert(fb_empty(hpdata->active_pages, HUGEPAGE_PAGES));
assert(hpdata->h_nactive == 0);
}
@@ -341,7 +341,7 @@ hpdata_assert_empty(const hpdata_t *hpdata) {
* match computed ones).
*/
static inline bool
hpdata_consistent(const hpdata_t *hpdata) {
hpdata_consistent(hpdata_t *hpdata) {
bool res = true;
const size_t active_urange_longest = fb_urange_longest(
@@ -432,60 +432,6 @@ void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age, bool is_huge);
void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
void hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz);
/*
* For buffering extent allocations we will perform out of
* a single ps.
*/
typedef struct hpdata_alloc_offset_s hpdata_alloc_offset_t;
struct hpdata_alloc_offset_s {
/*
* Index on the active bitmap for the extent to allocate.
* It is used to know which bits we'll need to set when we perform
* the allocation. They are in the range [index, index + npages).
*/
size_t index;
/*
* The length of the free bit range on the active bitmap,
* starting at index, before setting the bits in the range
* [index, index + npages).
* It is used to determine whether one of the allocations
* used up the longest free range on the active bitmap.
* If it did, we might have to update the longest free range
* metadata on the hpdata.
*/
size_t len_before;
/*
* The length of the longest free range in the range [0, index).
* When we need to update the longest free range on the hpdata,
* the new value is either longest_len (the max up to index),
* len_before - npages (what's left after we carve up the free
* range starting at index), or the max in the range
* [index + len_before, HUGEPAGE_PAGES), whichever is greater.
*/
size_t longest_len;
};
/*
* Given an hpdata that can serve an allocation request of size sz,
* find between one and max_nallocs offsets that can satisfy such
* an allocation request and buffer them in offsets (without actually
* reserving any space or updating hpdata). Return the number
* of offsets discovered.
*/
size_t hpdata_find_alloc_offsets(hpdata_t *hpdata, size_t sz,
hpdata_alloc_offset_t *offsets, size_t max_nallocs);
/* Reserve the allocation for the given offset. */
void *hpdata_reserve_alloc_offset(
hpdata_t *hpdata, size_t sz, hpdata_alloc_offset_t *offset);
/*
* Do any work that needs to be done after performing allocations
* from a single hpdata.
*/
void hpdata_post_reserve_alloc_offsets(hpdata_t *hpdata, size_t sz,
hpdata_alloc_offset_t *offsets, size_t nallocs);
/*
* The hpdata_purge_prepare_t allows grabbing the metadata required to purge
* subranges of a hugepage while holding a lock, drop the lock during the actual

18
include/jemalloc/internal/inspect.h
View File

@@ -11,7 +11,8 @@
*/
/*
* The following struct is for experimental purposes. See
* The following two structs are for experimental purposes. See
* experimental_utilization_query_ctl and
* experimental_utilization_batch_query_ctl in src/ctl.c.
*/
typedef struct inspect_extent_util_stats_s inspect_extent_util_stats_t;
@@ -21,7 +22,22 @@ struct inspect_extent_util_stats_s {
size_t size;
};
typedef struct inspect_extent_util_stats_verbose_s
inspect_extent_util_stats_verbose_t;
struct inspect_extent_util_stats_verbose_s {
void *slabcur_addr;
size_t nfree;
size_t nregs;
size_t size;
size_t bin_nfree;
size_t bin_nregs;
};
void inspect_extent_util_stats_get(
tsdn_t *tsdn, const void *ptr, size_t *nfree, size_t *nregs, size_t *size);
void inspect_extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
size_t *nfree, size_t *nregs, size_t *size, size_t *bin_nfree,
size_t *bin_nregs, void **slabcur_addr);
#endif /* JEMALLOC_INTERNAL_INSPECT_H */

8
include/jemalloc/internal/jemalloc_fork.h
View File

@@ -1,8 +0,0 @@
#ifndef JEMALLOC_INTERNAL_JEMALLOC_FORK_H
#define JEMALLOC_INTERNAL_JEMALLOC_FORK_H
void jemalloc_prefork(void);
void jemalloc_postfork_parent(void);
void jemalloc_postfork_child(void);
#endif /* JEMALLOC_INTERNAL_JEMALLOC_FORK_H */

40
include/jemalloc/internal/jemalloc_init.h
View File

@@ -1,40 +0,0 @@
#ifndef JEMALLOC_INTERNAL_JEMALLOC_INIT_H
#define JEMALLOC_INTERNAL_JEMALLOC_INIT_H
enum malloc_init_e {
malloc_init_uninitialized = 3,
malloc_init_a0_initialized = 2,
malloc_init_recursible = 1,
malloc_init_initialized = 0 /* Common case --> jnz. */
};
typedef enum malloc_init_e malloc_init_t;
extern malloc_init_t malloc_init_state;
bool malloc_is_initializer(void);
bool malloc_init_hard_a0(void);
bool malloc_init_hard(void);
JEMALLOC_ALWAYS_INLINE bool
malloc_init_a0(void) {
if (unlikely(malloc_init_state == malloc_init_uninitialized)) {
return malloc_init_hard_a0();
}
return false;
}
JEMALLOC_ALWAYS_INLINE bool
malloc_initialized(void) {
return (malloc_init_state == malloc_init_initialized);
}
JEMALLOC_ALWAYS_INLINE bool
malloc_init(void) {
if (unlikely(!malloc_initialized()) && malloc_init_hard()) {
return true;
}
return false;
}
#endif /* JEMALLOC_INTERNAL_JEMALLOC_INIT_H */

10
include/jemalloc/internal/jemalloc_internal_defs.h.in
View File

@@ -465,16 +465,6 @@
/* Is C++ support being built? */
#undef JEMALLOC_ENABLE_CXX
/* Are C++ exceptions enabled? */
#undef JEMALLOC_HAVE_CXX_EXCEPTIONS
/*
* If defined, throwing operator new aborts on OOM (with size logged) instead
* of throwing std::bad_alloc. Nothrow new still returns null. With LTO this
* lets the compiler prove operator new is no-throw and elide exception cleanup.
*/
#undef JEMALLOC_INFALLIBLE_NEW
/* Performs additional size checks when defined. */
#undef JEMALLOC_OPT_SIZE_CHECKS

31
include/jemalloc/internal/jemalloc_internal_externs.h
View File

@@ -1,6 +1,7 @@
#ifndef JEMALLOC_INTERNAL_EXTERNS_H
#define JEMALLOC_INTERNAL_EXTERNS_H
#include "jemalloc/internal/arena_types.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/fxp.h"
#include "jemalloc/internal/hpa_opts.h"
@@ -30,11 +31,13 @@ extern void (*JET_MUTABLE junk_alloc_callback)(void *ptr, size_t size);
extern void (*JET_MUTABLE invalid_conf_abort)(void);
extern bool opt_utrace;
extern bool opt_xmalloc;
extern bool opt_experimental_infallible_new;
extern bool opt_experimental_tcache_gc;
extern bool opt_zero;
extern unsigned opt_narenas;
extern fxp_t opt_narenas_ratio;
extern zero_realloc_action_t opt_zero_realloc_action;
extern malloc_init_t malloc_init_state;
extern const char *const zero_realloc_mode_names[];
extern atomic_zu_t zero_realloc_count;
extern bool opt_cache_oblivious;
@@ -44,7 +47,6 @@ extern bool opt_disable_large_size_classes;
extern const char *opt_malloc_conf_symlink;
extern const char *opt_malloc_conf_env_var;
extern const char *je_malloc_conf_2_conf_harder;
/* Escape free-fastpath when ptr & mask == 0 (for sanitization purpose). */
extern uintptr_t san_cache_bin_nonfast_mask;
@@ -52,9 +54,36 @@ extern uintptr_t san_cache_bin_nonfast_mask;
/* Number of CPUs. */
extern unsigned ncpus;
/* Number of arenas used for automatic multiplexing of threads and arenas. */
extern unsigned narenas_auto;
/* Base index for manual arenas. */
extern unsigned manual_arena_base;
/*
* Arenas that are used to service external requests. Not all elements of the
* arenas array are necessarily used; arenas are created lazily as needed.
*/
extern atomic_p_t arenas[];
extern unsigned huge_arena_ind;
void *a0malloc(size_t size);
void a0dalloc(void *ptr);
void *bootstrap_malloc(size_t size);
void *bootstrap_calloc(size_t num, size_t size);
void bootstrap_free(void *ptr);
void arena_set(unsigned ind, arena_t *arena);
unsigned narenas_total_get(void);
arena_t *arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
void arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena);
void iarena_cleanup(tsd_t *tsd);
void arena_cleanup(tsd_t *tsd);
size_t batch_alloc(void **ptrs, size_t num, size_t size, int flags);
void jemalloc_prefork(void);
void jemalloc_postfork_parent(void);
void jemalloc_postfork_child(void);
void sdallocx_default(void *ptr, size_t size, int flags);
void free_default(void *ptr);
void *malloc_default(size_t size);

84
include/jemalloc/internal/jemalloc_internal_includes.h Normal file
View File

@@ -0,0 +1,84 @@
#ifndef JEMALLOC_INTERNAL_INCLUDES_H
#define JEMALLOC_INTERNAL_INCLUDES_H
/*
* jemalloc can conceptually be broken into components (arena, tcache, etc.),
* but there are circular dependencies that cannot be broken without
* substantial performance degradation.
*
* Historically, we dealt with this by each header into four sections (types,
* structs, externs, and inlines), and included each header file multiple times
* in this file, picking out the portion we want on each pass using the
* following #defines:
* JEMALLOC_H_TYPES : Preprocessor-defined constants and pseudo-opaque data
* types.
* JEMALLOC_H_STRUCTS : Data structures.
* JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes.
* JEMALLOC_H_INLINES : Inline functions.
*
* We're moving toward a world in which the dependencies are explicit; each file
* will #include the headers it depends on (rather than relying on them being
* implicitly available via this file including every header file in the
* project).
*
* We're now in an intermediate state: we've broken up the header files to avoid
* having to include each one multiple times, but have not yet moved the
* dependency information into the header files (i.e. we still rely on the
* ordering in this file to ensure all a header's dependencies are available in
* its translation unit). Each component is now broken up into multiple header
* files, corresponding to the sections above (e.g. instead of "foo.h", we now
* have "foo_types.h", "foo_structs.h", "foo_externs.h", "foo_inlines.h").
*
* Those files which have been converted to explicitly include their
* inter-component dependencies are now in the initial HERMETIC HEADERS
* section. All headers may still rely on jemalloc_preamble.h (which, by fiat,
* must be included first in every translation unit) for system headers and
* global jemalloc definitions, however.
*/
/******************************************************************************/
/* TYPES */
/******************************************************************************/
#include "jemalloc/internal/arena_types.h"
#include "jemalloc/internal/tcache_types.h"
#include "jemalloc/internal/prof_types.h"
/******************************************************************************/
/* STRUCTS */
/******************************************************************************/
#include "jemalloc/internal/prof_structs.h"
#include "jemalloc/internal/arena_structs.h"
#include "jemalloc/internal/tcache_structs.h"
#include "jemalloc/internal/background_thread_structs.h"
/******************************************************************************/
/* EXTERNS */
/******************************************************************************/
#include "jemalloc/internal/jemalloc_internal_externs.h"
#include "jemalloc/internal/arena_externs.h"
#include "jemalloc/internal/large_externs.h"
#include "jemalloc/internal/tcache_externs.h"
#include "jemalloc/internal/prof_externs.h"
#include "jemalloc/internal/background_thread_externs.h"
/******************************************************************************/
/* INLINES */
/******************************************************************************/
#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
/*
* Include portions of arena code interleaved with tcache code in order to
* resolve circular dependencies.
*/
#include "jemalloc/internal/arena_inlines_a.h"
#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
#include "jemalloc/internal/tcache_inlines.h"
#include "jemalloc/internal/arena_inlines_b.h"
#include "jemalloc/internal/jemalloc_internal_inlines_c.h"
#include "jemalloc/internal/prof_inlines.h"
#include "jemalloc/internal/background_thread_inlines.h"
#endif /* JEMALLOC_INTERNAL_INCLUDES_H */

6
include/jemalloc/internal/jemalloc_internal_inlines_a.h
View File

@@ -2,13 +2,13 @@
#define JEMALLOC_INTERNAL_INLINES_A_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/arenas_management.h"
#include "jemalloc/internal/arena_externs.h"
#include "jemalloc/internal/arena_types.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/tcache_externs.h"
#include "jemalloc/internal/ticker.h"
JEMALLOC_ALWAYS_INLINE malloc_cpuid_t

106
include/jemalloc/internal/jemalloc_internal_inlines_b.h Normal file
View File

@@ -0,0 +1,106 @@
#ifndef JEMALLOC_INTERNAL_INLINES_B_H
#define JEMALLOC_INTERNAL_INLINES_B_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena_inlines_a.h"
#include "jemalloc/internal/extent.h"
#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
static inline void
percpu_arena_update(tsd_t *tsd, unsigned cpu) {
assert(have_percpu_arena);
arena_t *oldarena = tsd_arena_get(tsd);
assert(oldarena != NULL);
unsigned oldind = arena_ind_get(oldarena);
if (oldind != cpu) {
unsigned newind = cpu;
arena_t *newarena = arena_get(tsd_tsdn(tsd), newind, true);
assert(newarena != NULL);
/* Set new arena/tcache associations. */
arena_migrate(tsd, oldarena, newarena);
tcache_t *tcache = tcache_get(tsd);
if (tcache != NULL) {
tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
assert(tcache_slow->arena != NULL);
tcache_arena_reassociate(
tsd_tsdn(tsd), tcache_slow, tcache, newarena);
}
}
}
/* Choose an arena based on a per-thread value. */
static inline arena_t *
arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
arena_t *ret;
if (arena != NULL) {
return arena;
}
/* During reentrancy, arena 0 is the safest bet. */
if (unlikely(tsd_reentrancy_level_get(tsd) > 0)) {
return arena_get(tsd_tsdn(tsd), 0, true);
}
ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd);
if (unlikely(ret == NULL)) {
ret = arena_choose_hard(tsd, internal);
assert(ret);
if (tcache_available(tsd)) {
tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
tcache_t *tcache = tsd_tcachep_get(tsd);
if (tcache_slow->arena != NULL) {
/* See comments in tsd_tcache_data_init().*/
assert(tcache_slow->arena
== arena_get(tsd_tsdn(tsd), 0, false));
if (tcache_slow->arena != ret) {
tcache_arena_reassociate(tsd_tsdn(tsd),
tcache_slow, tcache, ret);
}
} else {
tcache_arena_associate(
tsd_tsdn(tsd), tcache_slow, tcache, ret);
}
}
}
/*
* Note that for percpu arena, if the current arena is outside of the
* auto percpu arena range, (i.e. thread is assigned to a manually
* managed arena), then percpu arena is skipped.
*/
if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)
&& !internal
&& (arena_ind_get(ret) < percpu_arena_ind_limit(opt_percpu_arena))
&& (ret->last_thd != tsd_tsdn(tsd))) {
unsigned ind = percpu_arena_choose();
if (arena_ind_get(ret) != ind) {
percpu_arena_update(tsd, ind);
ret = tsd_arena_get(tsd);
}
ret->last_thd = tsd_tsdn(tsd);
}
return ret;
}
static inline arena_t *
arena_choose(tsd_t *tsd, arena_t *arena) {
return arena_choose_impl(tsd, arena, false);
}
static inline arena_t *
arena_ichoose(tsd_t *tsd, arena_t *arena) {
return arena_choose_impl(tsd, arena, true);
}
static inline bool
arena_is_auto(arena_t *arena) {
assert(narenas_auto > 0);
return (arena_ind_get(arena) < manual_arena_base);
}
#endif /* JEMALLOC_INTERNAL_INLINES_B_H */

55
include/jemalloc/internal/jemalloc_internal_inlines_c.h
View File

@@ -2,13 +2,12 @@
#define JEMALLOC_INTERNAL_INLINES_C_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/arena_inlines.h"
#include "jemalloc/internal/arena_externs.h"
#include "jemalloc/internal/arena_inlines_b.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/jemalloc_init.h"
#include "jemalloc/internal/hook.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/log.h"
#include "jemalloc/internal/malloc_dispatch_inlines.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/thread_event.h"
#include "jemalloc/internal/witness.h"
@@ -68,7 +67,7 @@ iallocztm_explicit_slab(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
}
ret = malloc_dispatch_malloc(
ret = arena_malloc(
tsdn, arena, size, ind, zero, slab, tcache, slow_path);
if (config_stats && is_internal && likely(ret != NULL)) {
arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@@ -103,8 +102,7 @@ ipallocztm_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment,
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
ret = malloc_dispatch_palloc(
tsdn, arena, usize, alignment, zero, slab, tcache);
ret = arena_palloc(tsdn, arena, usize, alignment, zero, slab, tcache);
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
if (config_stats && is_internal && likely(ret != NULL)) {
arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@@ -158,7 +156,7 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
&& tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) {
assert(tcache == NULL);
}
malloc_dispatch_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
}
JEMALLOC_ALWAYS_INLINE void
@@ -171,12 +169,13 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
emap_alloc_ctx_t *alloc_ctx, bool slow_path) {
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
malloc_dispatch_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
}
JEMALLOC_ALWAYS_INLINE void *
iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena) {
size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena,
hook_ralloc_args_t *hook_args) {
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
void *p;
@@ -197,13 +196,27 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
*/
copysize = (size < oldsize) ? size : oldsize;
memcpy(p, ptr, copysize);
hook_invoke_alloc(
hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx, p,
(uintptr_t)p, hook_args->args);
hook_invoke_dalloc(
hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx,
ptr, hook_args->args);
isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
return p;
}
/*
* is_realloc threads through the knowledge of whether or not this call comes
* from je_realloc (as opposed to je_rallocx); this ensures that we pass the
* correct entry point into any hooks.
* Note that these functions are all force-inlined, so no actual bool gets
* passed-around anywhere.
*/
JEMALLOC_ALWAYS_INLINE void *
iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena) {
size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena,
hook_ralloc_args_t *hook_args) {
assert(ptr != NULL);
assert(size != 0);
witness_assert_depth_to_rank(
@@ -216,26 +229,27 @@ iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
* and copy.
*/
return iralloct_realign(tsdn, ptr, oldsize, size, alignment,
zero, slab, tcache, arena);
zero, slab, tcache, arena, hook_args);
}
return malloc_dispatch_ralloc(
tsdn, arena, ptr, oldsize, size, alignment, zero, slab, tcache);
return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero,
slab, tcache, hook_args);
}
JEMALLOC_ALWAYS_INLINE void *
iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment,
size_t usize, bool zero, tcache_t *tcache, arena_t *arena) {
size_t usize, bool zero, tcache_t *tcache, arena_t *arena,
hook_ralloc_args_t *hook_args) {
bool slab = sz_can_use_slab(usize);
return iralloct_explicit_slab(tsdn, ptr, oldsize, size, alignment, zero,
slab, tcache, arena);
slab, tcache, arena, hook_args);
}
JEMALLOC_ALWAYS_INLINE void *
iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
size_t usize, bool zero) {
size_t usize, bool zero, hook_ralloc_args_t *hook_args) {
return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, usize,
zero, tcache_get(tsd), NULL);
zero, tcache_get(tsd), NULL, hook_args);
}
JEMALLOC_ALWAYS_INLINE bool
@@ -266,6 +280,11 @@ fastpath_success_finish(
}
}
JEMALLOC_ALWAYS_INLINE bool
malloc_initialized(void) {
return (malloc_init_state == malloc_init_initialized);
}
/*
* malloc() fastpath. Included here so that we can inline it into operator new;
* function call overhead there is non-negligible as a fraction of total CPU in

8
include/jemalloc/internal/jemalloc_internal_types.h
View File

@@ -20,6 +20,14 @@ typedef enum zero_realloc_action_e zero_realloc_action_t;
/* Signature of write callback. */
typedef void(write_cb_t)(void *, const char *);
enum malloc_init_e {
malloc_init_uninitialized = 3,
malloc_init_a0_initialized = 2,
malloc_init_recursible = 1,
malloc_init_initialized = 0 /* Common case --> jnz. */
};
typedef enum malloc_init_e malloc_init_t;
/*
* Flags bits:
*

12
include/jemalloc/internal/jemalloc_preamble.h.in
View File

@@ -238,18 +238,6 @@ static const bool config_enable_cxx =
#endif
;
/*
* Whether the throwing operator new aborts on OOM instead of throwing
* std::bad_alloc (--enable-cxx-infallible-new).
*/
static const bool config_infallible_new =
#ifdef JEMALLOC_INFALLIBLE_NEW
true
#else
false
#endif
;
#if defined(_WIN32) || defined(__APPLE__) || defined(JEMALLOC_HAVE_SCHED_GETCPU)
/* Currently percpu_arena depends on sched_getcpu. */
#define JEMALLOC_PERCPU_ARENA

14
include/jemalloc/internal/large.h → include/jemalloc/internal/large_externs.h
View File

@@ -1,12 +1,9 @@
#ifndef JEMALLOC_INTERNAL_LARGE_H
#define JEMALLOC_INTERNAL_LARGE_H
#ifndef JEMALLOC_INTERNAL_LARGE_EXTERNS_H
#define JEMALLOC_INTERNAL_LARGE_EXTERNS_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/edata.h"
/* Forward decls; only used as pointer types below. */
typedef struct arena_s arena_t;
typedef struct prof_info_s prof_info_t;
#include "jemalloc/internal/hook.h"
void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
void *large_palloc(
@@ -14,7 +11,8 @@ void *large_palloc(
bool large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min,
size_t usize_max, bool zero);
void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
size_t alignment, bool zero, tcache_t *tcache);
size_t alignment, bool zero, tcache_t *tcache,
hook_ralloc_args_t *hook_args);
void large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata);
void large_dalloc_finish(tsdn_t *tsdn, edata_t *edata);
@@ -24,4 +22,4 @@ void large_prof_info_get(
void large_prof_tctx_reset(edata_t *edata);
void large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size);
#endif /* JEMALLOC_INTERNAL_LARGE_H */
#endif /* JEMALLOC_INTERNAL_LARGE_EXTERNS_H */

19
include/jemalloc/internal/malloc_dispatch.h
View File

@@ -1,19 +0,0 @@
#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_H
#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/tsd_types.h"
/* Forward decls; only used as pointer types below. */
typedef struct arena_s arena_t;
typedef struct tcache_s tcache_t;
void *malloc_dispatch_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero, bool slab, tcache_t *tcache);
void malloc_dispatch_dalloc_promoted(
tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
void *malloc_dispatch_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr,
size_t oldsize, size_t size, size_t alignment, bool zero, bool slab,
tcache_t *tcache);
#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_H */

276
include/jemalloc/internal/malloc_dispatch_inlines.h
View File

@@ -1,276 +0,0 @@
#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/arena_inlines.h"
#include "jemalloc/internal/bin_inlines.h"
#include "jemalloc/internal/div.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/large.h"
#include "jemalloc/internal/malloc_dispatch.h"
#include "jemalloc/internal/safety_check.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/tcache_inlines.h"
JEMALLOC_ALWAYS_INLINE void *
malloc_dispatch_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
bool zero, bool slab, tcache_t *tcache, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
if (likely(tcache != NULL)) {
if (likely(slab)) {
assert(sz_can_use_slab(size));
return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
size, ind, zero, slow_path);
} else if (likely(tcache_can_cache_large(tcache, ind))) {
return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
size, ind, zero, slow_path);
}
/* (size > tcache_max) case falls through. */
}
return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
}
static inline void
malloc_dispatch_dalloc_large_no_tcache(
tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
/*
* szind both classifies small vs large and validates the extent --
* inactive extents have szind == SC_NSIZES.
*/
if (config_prof && unlikely(szind < SC_NBINS)) {
malloc_dispatch_dalloc_promoted(tsdn, ptr, NULL, true);
} else {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
if (large_dalloc_safety_checks(edata, ptr, usize)) {
/* See the comment in isfree. */
return;
}
large_dalloc(tsdn, edata);
}
}
static inline void
malloc_dispatch_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
assert(ptr != NULL);
emap_alloc_ctx_t alloc_ctx;
emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.szind < SC_NSIZES);
assert(alloc_ctx.slab == edata_slab_get(edata));
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
== edata_usize_get(edata));
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
arena_dalloc_small(tsdn, ptr);
} else {
malloc_dispatch_dalloc_large_no_tcache(
tsdn, ptr, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx));
}
}
JEMALLOC_ALWAYS_INLINE void
malloc_dispatch_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
szind_t szind, size_t usize, bool slow_path) {
assert(!tsdn_null(tsdn) && tcache != NULL);
bool is_sample_promoted = config_prof && szind < SC_NBINS;
if (unlikely(is_sample_promoted)) {
malloc_dispatch_dalloc_promoted(tsdn, ptr, tcache, slow_path);
} else {
if (tcache_can_cache_large(tcache, szind)) {
tcache_dalloc_large(
tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
} else {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
if (large_dalloc_safety_checks(edata, ptr, usize)) {
/* See the comment in isfree. */
return;
}
large_dalloc(tsdn, edata);
}
}
}
JEMALLOC_ALWAYS_INLINE bool
malloc_dispatch_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
if (!config_debug) {
return false;
}
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
szind_t binind = edata_szind_get(edata);
div_info_t div_info = arena_binind_div_info[binind];
/*
* Calls the internal function bin_slab_regind_impl because the
* safety check does not require a lock.
*/
size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
slab_data_t *slab_data = edata_slab_data_get(edata);
const bin_info_t *bin_info = &bin_infos[binind];
assert(edata_nfree_get(edata) < bin_info->nregs);
if (unlikely(!bitmap_get(
slab_data->bitmap, &bin_info->bitmap_info, regind))) {
safety_check_fail(
"Invalid deallocation detected: the pointer being freed (%p) not "
"currently active, possibly caused by double free bugs.\n",
ptr);
return true;
}
return false;
}
JEMALLOC_ALWAYS_INLINE void
malloc_dispatch_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL);
if (unlikely(tcache == NULL)) {
malloc_dispatch_dalloc_no_tcache(tsdn, ptr);
return;
}
emap_alloc_ctx_t alloc_ctx;
if (caller_alloc_ctx != NULL) {
alloc_ctx = *caller_alloc_ctx;
} else {
util_assume(tsdn != NULL);
emap_alloc_ctx_lookup(
tsdn, &arena_emap_global, ptr, &alloc_ctx);
}
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.szind < SC_NSIZES);
assert(alloc_ctx.slab == edata_slab_get(edata));
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
== edata_usize_get(edata));
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
return;
}
tcache_dalloc_small(
tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
} else {
malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
}
}
static inline void
malloc_dispatch_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
assert(ptr != NULL);
assert(size <= SC_LARGE_MAXCLASS);
emap_alloc_ctx_t alloc_ctx;
if (!config_prof || !opt_prof) {
/*
* There is no risk of being confused by a promoted sampled
* object, so base szind and slab on the given size.
*/
szind_t szind = sz_size2index(size);
emap_alloc_ctx_init(
&alloc_ctx, szind, (szind < SC_NBINS), size);
}
if ((config_prof && opt_prof) || config_debug) {
emap_alloc_ctx_lookup(
tsdn, &arena_emap_global, ptr, &alloc_ctx);
assert(alloc_ctx.szind == sz_size2index(size));
assert((config_prof && opt_prof)
|| alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.slab == edata_slab_get(edata));
}
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
arena_dalloc_small(tsdn, ptr);
} else {
malloc_dispatch_dalloc_large_no_tcache(
tsdn, ptr, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx));
}
}
JEMALLOC_ALWAYS_INLINE void
malloc_dispatch_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL);
assert(size <= SC_LARGE_MAXCLASS);
if (unlikely(tcache == NULL)) {
malloc_dispatch_sdalloc_no_tcache(tsdn, ptr, size);
return;
}
emap_alloc_ctx_t alloc_ctx;
if (config_prof && opt_prof) {
if (caller_alloc_ctx == NULL) {
/* Uncommon case and should be a static check. */
emap_alloc_ctx_lookup(
tsdn, &arena_emap_global, ptr, &alloc_ctx);
assert(alloc_ctx.szind == sz_size2index(size));
assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
} else {
alloc_ctx = *caller_alloc_ctx;
}
} else {
/*
* There is no risk of being confused by a promoted sampled
* object, so base szind and slab on the given size.
*/
alloc_ctx.szind = sz_size2index(size);
alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
}
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.slab == edata_slab_get(edata));
emap_alloc_ctx_init(
&alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
== edata_usize_get(edata));
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
return;
}
tcache_dalloc_small(
tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
} else {
malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
sz_s2u(size), slow_path);
}
}
#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H */

1
include/jemalloc/internal/mutex_prof.h
View File

@@ -30,7 +30,6 @@ typedef enum {
OP(extents_dirty) \
OP(extents_muzzy) \
OP(extents_retained) \
OP(extents_pinned) \
OP(decay_dirty) \
OP(decay_muzzy) \
OP(base) \

20
include/jemalloc/internal/pa.h
View File

@@ -10,13 +10,15 @@
#include "jemalloc/internal/hpa.h"
#include "jemalloc/internal/lockedint.h"
#include "jemalloc/internal/pac.h"
#include "jemalloc/internal/pai.h"
#include "jemalloc/internal/sec.h"
/*
* The page allocator; responsible for acquiring pages of memory for
* allocations. It dispatches each page-level allocation request to either
* the PAC ("page allocator classic") or the HPA ("hugepage allocator")
* by calling their pac_*() / hpa_*() entry points directly.
* allocations. It picks the implementation of the page allocator interface
* (i.e. a pai_t) to handle a given page-level allocation request. For now, the
* only such implementation is the PAC code ("page allocator classic"), but
* others will be coming soon.
*/
typedef struct pa_central_s pa_central_t;
@@ -94,7 +96,7 @@ struct pa_shard_s {
/* Allocates from a PAC. */
pac_t pac;
hpa_shard_t hpa;
hpa_shard_t hpa_shard;
/* The source of edata_t objects. */
edata_cache_t edata_cache;
@@ -118,7 +120,7 @@ pa_shard_dont_decay_muzzy(pa_shard_t *shard) {
}
static inline ehooks_t *
pa_shard_ehooks_get(const pa_shard_t *shard) {
pa_shard_ehooks_get(pa_shard_t *shard) {
return base_ehooks_get(shard->base);
}
@@ -219,12 +221,12 @@ void pa_shard_prefork5(tsdn_t *tsdn, pa_shard_t *shard);
void pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard);
void pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard);
size_t pa_shard_nactive(const pa_shard_t *shard);
size_t pa_shard_ndirty(const pa_shard_t *shard);
size_t pa_shard_nmuzzy(const pa_shard_t *shard);
size_t pa_shard_nactive(pa_shard_t *shard);
size_t pa_shard_ndirty(pa_shard_t *shard);
size_t pa_shard_nmuzzy(pa_shard_t *shard);
void pa_shard_basic_stats_merge(
const pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,

50
include/jemalloc/internal/pac.h
View File

@@ -4,11 +4,10 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/decay.h"
#include "jemalloc/internal/ecache.h"
#include "jemalloc/internal/edata.h"
#include "jemalloc/internal/edata_cache.h"
#include "jemalloc/internal/exp_grow.h"
#include "jemalloc/internal/lockedint.h"
#include "jemalloc/internal/tsd_types.h"
#include "jemalloc/internal/pai.h"
#include "san_bump.h"
/*
@@ -52,8 +51,6 @@ struct pac_estats_s {
size_t muzzy_bytes;
size_t nretained;
size_t retained_bytes;
size_t npinned;
size_t pinned_bytes;
};
typedef struct pac_stats_s pac_stats_t;
@@ -64,14 +61,9 @@ struct pac_stats_s {
/*
* Number of unused virtual memory bytes currently retained. Retained
* bytes are technically mapped (though always decommitted or purged),
* but they are excluded from pac_mapped.
* but they are excluded from the mapped statistic (above).
*/
size_t retained; /* Derived. */
/*
* Number of bytes in pinned (non-reclaimable) extents currently
* cached. Unlike retained, pinned bytes count toward pac_mapped.
*/
size_t pinned; /* Derived. */
/*
* Number of bytes currently mapped, excluding retained memory (and any
@@ -88,8 +80,11 @@ struct pac_stats_s {
typedef struct pac_s pac_t;
struct pac_s {
/* True once pinned memory has been seen. */
atomic_b_t has_pinned;
/*
* Must be the first member (we convert it to a PAC given only a
* pointer). The handle to the allocation interface.
*/
pai_t pai;
/*
* Collections of extents that were previously allocated. These are
* used when allocating extents, in an attempt to re-use address space.
@@ -99,7 +94,6 @@ struct pac_s {
ecache_t ecache_dirty;
ecache_t ecache_muzzy;
ecache_t ecache_retained;
ecache_t ecache_pinned;
base_t *base;
emap_t *emap;
@@ -161,39 +155,13 @@ bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats,
malloc_mutex_t *stats_mtx);
edata_t *pac_alloc(tsdn_t *tsdn, pac_t *pac, size_t size, size_t alignment,
bool zero, bool guarded, bool frequent_reuse,
bool *deferred_work_generated);
bool pac_expand(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
size_t new_size, bool zero, bool *deferred_work_generated);
bool pac_shrink(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
size_t new_size, bool *deferred_work_generated);
void pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata,
bool *deferred_work_generated);
uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pac_t *pac);
static inline size_t
pac_mapped(const pac_t *pac) {
pac_mapped(pac_t *pac) {
return atomic_load_zu(&pac->stats->pac_mapped, ATOMIC_RELAXED);
}
void extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
ecache_t *ecache, edata_t *edata);
static inline void
pac_record_grown(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
edata_t *edata) {
bool pinned = edata_pinned_get(edata);
if (pinned && config_stats) {
atomic_fetch_add_zu(&pac->stats->pac_mapped,
edata_size_get(edata), ATOMIC_RELAXED);
}
extent_record(tsdn, pac, ehooks,
pinned ? &pac->ecache_pinned : &pac->ecache_retained, edata);
}
static inline ehooks_t *
pac_ehooks_get(const pac_t *pac) {
pac_ehooks_get(pac_t *pac) {
return base_ehooks_get(pac->base);
}

63
include/jemalloc/internal/pai.h Normal file
View File

@@ -0,0 +1,63 @@
#ifndef JEMALLOC_INTERNAL_PAI_H
#define JEMALLOC_INTERNAL_PAI_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/edata.h"
#include "jemalloc/internal/tsd_types.h"
/* An interface for page allocation. */
typedef struct pai_s pai_t;
struct pai_s {
/* Returns NULL on failure. */
edata_t *(*alloc)(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool guarded, bool frequent_reuse,
bool *deferred_work_generated);
bool (*expand)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero,
bool *deferred_work_generated);
bool (*shrink)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool *deferred_work_generated);
void (*dalloc)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated);
uint64_t (*time_until_deferred_work)(tsdn_t *tsdn, pai_t *self);
};
/*
* These are just simple convenience functions to avoid having to reference the
* same pai_t twice on every invocation.
*/
static inline edata_t *
pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
return self->alloc(tsdn, self, size, alignment, zero, guarded,
frequent_reuse, deferred_work_generated);
}
static inline bool
pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool zero, bool *deferred_work_generated) {
return self->expand(tsdn, self, edata, old_size, new_size, zero,
deferred_work_generated);
}
static inline bool
pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool *deferred_work_generated) {
return self->shrink(
tsdn, self, edata, old_size, new_size, deferred_work_generated);
}
static inline void
pai_dalloc(
tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
self->dalloc(tsdn, self, edata, deferred_work_generated);
}
static inline uint64_t
pai_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
return self->time_until_deferred_work(tsdn, self);
}
#endif /* JEMALLOC_INTERNAL_PAI_H */

1
include/jemalloc/internal/peak_event.h
View File

@@ -2,7 +2,6 @@
#define JEMALLOC_INTERNAL_PEAK_EVENT_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/thread_event_registry.h"
#include "jemalloc/internal/tsd_types.h"
/*

427
include/jemalloc/internal/prof.h
View File

@@ -1,427 +0,0 @@
#ifndef JEMALLOC_INTERNAL_PROF_H
#define JEMALLOC_INTERNAL_PROF_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/ckh.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/prof_hook.h"
#include "jemalloc/internal/rb.h"
#include "jemalloc/internal/thread_event_registry.h"
/* Forward decl; only base_t * is used as a pointer arg below. */
typedef struct base_s base_t;
/******************************************************************************/
/* TYPES */
/******************************************************************************/
typedef struct prof_bt_s prof_bt_t;
typedef struct prof_cnt_s prof_cnt_t;
typedef struct prof_tctx_s prof_tctx_t;
typedef struct prof_info_s prof_info_t;
typedef struct prof_gctx_s prof_gctx_t;
typedef struct prof_tdata_s prof_tdata_t;
typedef struct prof_recent_s prof_recent_t;
/* Option defaults. */
#ifdef JEMALLOC_PROF
# define PROF_PREFIX_DEFAULT "jeprof"
#else
# define PROF_PREFIX_DEFAULT ""
#endif
#define LG_PROF_SAMPLE_DEFAULT 19
#define LG_PROF_INTERVAL_DEFAULT -1
/*
* Hard limit on stack backtrace depth. The version of prof_backtrace() that
* is based on __builtin_return_address() necessarily has a hard-coded number
* of backtrace frame handlers, and should be kept in sync with this setting.
*/
#ifdef JEMALLOC_PROF_GCC
# define PROF_BT_MAX_LIMIT 256
#else
# define PROF_BT_MAX_LIMIT UINT_MAX
#endif
#define PROF_BT_MAX_DEFAULT 128
/* Initial hash table size. */
#define PROF_CKH_MINITEMS 64
/* Size of memory buffer to use when writing dump files. */
#ifndef JEMALLOC_PROF
/* Minimize memory bloat for non-prof builds. */
# define PROF_DUMP_BUFSIZE 1
#elif defined(JEMALLOC_DEBUG)
/* Use a small buffer size in debug build, mainly to facilitate testing. */
# define PROF_DUMP_BUFSIZE 16
#else
# define PROF_DUMP_BUFSIZE 65536
#endif
/* Size of size class related tables */
#ifdef JEMALLOC_PROF
# define PROF_SC_NSIZES SC_NSIZES
#else
/* Minimize memory bloat for non-prof builds. */
# define PROF_SC_NSIZES 1
#endif
/* Size of stack-allocated buffer used by prof_printf(). */
#define PROF_PRINTF_BUFSIZE 128
/*
* Number of mutexes shared among all gctx's. No space is allocated for these
* unless profiling is enabled, so it's okay to over-provision.
*/
#define PROF_NCTX_LOCKS 1024
/*
* Number of mutexes shared among all tdata's. No space is allocated for these
* unless profiling is enabled, so it's okay to over-provision.
*/
#define PROF_NTDATA_LOCKS 256
/* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF
# define PROF_DUMP_FILENAME_LEN (PATH_MAX + 1)
#else
# define PROF_DUMP_FILENAME_LEN 1
#endif
/* Default number of recent allocations to record. */
#define PROF_RECENT_ALLOC_MAX_DEFAULT 0
/* Thread name storage size limit. */
#define PROF_THREAD_NAME_MAX_LEN 16
/*
* Minimum required alignment for sampled allocations. Over-aligning sampled
* allocations allows us to quickly identify them on the dalloc path without
* resorting to metadata lookup.
*/
#define PROF_SAMPLE_ALIGNMENT PAGE
#define PROF_SAMPLE_ALIGNMENT_MASK PAGE_MASK
/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
#define PROF_TCTX_SENTINEL ((prof_tctx_t *)((uintptr_t)1U))
/******************************************************************************/
/* STRUCTS */
/******************************************************************************/
struct prof_bt_s {
/* Backtrace, stored as len program counters. */
void **vec;
unsigned len;
};
#ifdef JEMALLOC_PROF_LIBGCC
/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */
typedef struct {
void **vec;
unsigned *len;
unsigned max;
} prof_unwind_data_t;
#endif
struct prof_cnt_s {
/* Profiling counters. */
uint64_t curobjs;
uint64_t curobjs_shifted_unbiased;
uint64_t curbytes;
uint64_t curbytes_unbiased;
uint64_t accumobjs;
uint64_t accumobjs_shifted_unbiased;
uint64_t accumbytes;
uint64_t accumbytes_unbiased;
};
typedef enum {
prof_tctx_state_initializing,
prof_tctx_state_nominal,
prof_tctx_state_dumping,
prof_tctx_state_purgatory /* Dumper must finish destroying. */
} prof_tctx_state_t;
struct prof_tctx_s {
/* Thread data for thread that performed the allocation. */
prof_tdata_t *tdata;
/*
* Copy of tdata->thr_{uid,discrim}, necessary because tdata may be
* defunct during teardown.
*/
uint64_t thr_uid;
uint64_t thr_discrim;
/*
* Reference count of how many times this tctx object is referenced in
* recent allocation / deallocation records, protected by tdata->lock.
*/
uint64_t recent_count;
/* Profiling counters, protected by tdata->lock. */
prof_cnt_t cnts;
/* Associated global context. */
prof_gctx_t *gctx;
/*
* UID that distinguishes multiple tctx's created by the same thread,
* but coexisting in gctx->tctxs. There are two ways that such
* coexistence can occur:
* - A dumper thread can cause a tctx to be retained in the purgatory
* state.
* - Although a single "producer" thread must create all tctx's which
* share the same thr_uid, multiple "consumers" can each concurrently
* execute portions of prof_tctx_destroy(). prof_tctx_destroy() only
* gets called once each time cnts.cur{objs,bytes} drop to 0, but this
* threshold can be hit again before the first consumer finishes
* executing prof_tctx_destroy().
*/
uint64_t tctx_uid;
/* Linkage into gctx's tctxs. */
rb_node(prof_tctx_t) tctx_link;
/*
* True during prof_alloc_prep()..prof_malloc_sample_object(), prevents
* sample vs destroy race.
*/
bool prepared;
/* Current dump-related state, protected by gctx->lock. */
prof_tctx_state_t state;
/*
* Copy of cnts snapshotted during early dump phase, protected by
* dump_mtx.
*/
prof_cnt_t dump_cnts;
};
typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;
struct prof_info_s {
/* Time when the allocation was made. */
nstime_t alloc_time;
/* Points to the prof_tctx_t corresponding to the allocation. */
prof_tctx_t *alloc_tctx;
/* Allocation request size. */
size_t alloc_size;
};
struct prof_gctx_s {
/* Protects nlimbo, cnt_summed, and tctxs. */
malloc_mutex_t *lock;
/*
* Number of threads that currently cause this gctx to be in a state of
* limbo due to one of:
* - Initializing this gctx.
* - Initializing per thread counters associated with this gctx.
* - Preparing to destroy this gctx.
* - Dumping a heap profile that includes this gctx.
* nlimbo must be 1 (single destroyer) in order to safely destroy the
* gctx.
*/
unsigned nlimbo;
/*
* Tree of profile counters, one for each thread that has allocated in
* this context.
*/
prof_tctx_tree_t tctxs;
/* Linkage for tree of contexts to be dumped. */
rb_node(prof_gctx_t) dump_link;
/* Temporary storage for summation during dump. */
prof_cnt_t cnt_summed;
/* Associated backtrace. */
prof_bt_t bt;
/* Backtrace vector, variable size, referred to by bt. */
void *vec[1];
};
typedef rb_tree(prof_gctx_t) prof_gctx_tree_t;
struct prof_tdata_s {
malloc_mutex_t *lock;
/* Monotonically increasing unique thread identifier. */
uint64_t thr_uid;
/*
* Monotonically increasing discriminator among tdata structures
* associated with the same thr_uid.
*/
uint64_t thr_discrim;
rb_node(prof_tdata_t) tdata_link;
/*
* Counter used to initialize prof_tctx_t's tctx_uid. No locking is
* necessary when incrementing this field, because only one thread ever
* does so.
*/
uint64_t tctx_uid_next;
/*
* Hash of (prof_bt_t *)-->(prof_tctx_t *). Each thread tracks
* backtraces for which it has non-zero allocation/deallocation counters
* associated with thread-specific prof_tctx_t objects. Other threads
* may write to prof_tctx_t contents when freeing associated objects.
*/
ckh_t bt2tctx;
/* Included in heap profile dumps if has content. */
char thread_name[PROF_THREAD_NAME_MAX_LEN];
/* State used to avoid dumping while operating on prof internals. */
bool enq;
bool enq_idump;
bool enq_gdump;
/*
* Set to true during an early dump phase for tdata's which are
* currently being dumped. New threads' tdata's have this initialized
* to false so that they aren't accidentally included in later dump
* phases.
*/
bool dumping;
/*
* True if profiling is active for this tdata's thread
* (thread.prof.active mallctl).
*/
bool active;
bool attached;
bool expired;
/* Temporary storage for summation during dump. */
prof_cnt_t cnt_summed;
/* Backtrace vector, used for calls to prof_backtrace(). */
void **vec;
};
typedef rb_tree(prof_tdata_t) prof_tdata_tree_t;
struct prof_recent_s {
nstime_t alloc_time;
nstime_t dalloc_time;
ql_elm(prof_recent_t) link;
size_t size;
size_t usize;
atomic_p_t alloc_edata; /* NULL means allocation has been freed. */
prof_tctx_t *alloc_tctx;
prof_tctx_t *dalloc_tctx;
};
/******************************************************************************/
/* EXTERNS */
/******************************************************************************/
extern bool opt_prof;
extern bool opt_prof_active;
extern bool opt_prof_thread_active_init;
extern unsigned opt_prof_bt_max;
extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */
extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */
extern bool opt_prof_gdump; /* High-water memory dumping. */
extern bool opt_prof_final; /* Final profile dumping. */
extern bool opt_prof_leak; /* Dump leak summary at exit. */
extern bool opt_prof_leak_error; /* Exit with error code if memory leaked */
extern bool opt_prof_accum; /* Report cumulative bytes. */
extern bool opt_prof_log; /* Turn logging on at boot. */
extern char opt_prof_prefix[
/* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF
PATH_MAX +
#endif
1];
extern bool opt_prof_unbias;
/* Include pid namespace in profile file names. */
extern bool opt_prof_pid_namespace;
/* For recording recent allocations */
extern ssize_t opt_prof_recent_alloc_max;
/* Whether to use thread name provided by the system or by mallctl. */
extern bool opt_prof_sys_thread_name;
/* Whether to record per size class counts and request size totals. */
extern bool opt_prof_stats;
/* Accessed via prof_active_[gs]et{_unlocked,}(). */
extern bool prof_active_state;
/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */
extern bool prof_gdump_val;
/* Profile dump interval, measured in bytes allocated. */
extern uint64_t prof_interval;
/*
* Initialized as opt_lg_prof_sample, and potentially modified during profiling
* resets.
*/
extern size_t lg_prof_sample;
extern bool prof_booted;
void prof_backtrace_hook_set(prof_backtrace_hook_t hook);
prof_backtrace_hook_t prof_backtrace_hook_get(void);
void prof_dump_hook_set(prof_dump_hook_t hook);
prof_dump_hook_t prof_dump_hook_get(void);
void prof_sample_hook_set(prof_sample_hook_t hook);
prof_sample_hook_t prof_sample_hook_get(void);
void prof_sample_free_hook_set(prof_sample_free_hook_t hook);
prof_sample_free_hook_t prof_sample_free_hook_get(void);
/* Functions only accessed in prof_inlines.h */
prof_tdata_t *prof_tdata_init(tsd_t *tsd);
prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx);
void prof_malloc_sample_object(
tsd_t *tsd, const void *ptr, size_t size, size_t usize, prof_tctx_t *tctx);
void prof_free_sampled_object(
tsd_t *tsd, const void *ptr, size_t usize, prof_info_t *prof_info);
prof_tctx_t *prof_tctx_create(tsd_t *tsd);
void prof_idump(tsdn_t *tsdn);
bool prof_mdump(tsd_t *tsd, const char *filename);
void prof_gdump(tsdn_t *tsdn);
void prof_tdata_cleanup(tsd_t *tsd);
bool prof_active_get(tsdn_t *tsdn);
bool prof_active_set(tsdn_t *tsdn, bool active);
const char *prof_thread_name_get(tsd_t *tsd);
int prof_thread_name_set(tsd_t *tsd, const char *thread_name);
bool prof_thread_active_get(tsd_t *tsd);
bool prof_thread_active_set(tsd_t *tsd, bool active);
bool prof_thread_active_init_get(tsdn_t *tsdn);
bool prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
bool prof_gdump_get(tsdn_t *tsdn);
bool prof_gdump_set(tsdn_t *tsdn, bool active);
void prof_boot0(void);
void prof_boot1(void);
bool prof_boot2(tsd_t *tsd, base_t *base);
void prof_prefork0(tsdn_t *tsdn);
void prof_prefork1(tsdn_t *tsdn);
void prof_postfork_parent(tsdn_t *tsdn);
void prof_postfork_child(tsdn_t *tsdn);
uint64_t prof_sample_new_event_wait(tsd_t *tsd);
uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd);
extern te_base_cb_t prof_sample_te_handler;
#endif /* JEMALLOC_INTERNAL_PROF_H */

165
include/jemalloc/internal/prof_externs.h Normal file
View File

@@ -0,0 +1,165 @@
#ifndef JEMALLOC_INTERNAL_PROF_EXTERNS_H
#define JEMALLOC_INTERNAL_PROF_EXTERNS_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/base.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/prof_hook.h"
#include "jemalloc/internal/thread_event_registry.h"
extern bool opt_prof;
extern bool opt_prof_active;
extern bool opt_prof_thread_active_init;
extern unsigned opt_prof_bt_max;
extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */
extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */
extern bool opt_prof_gdump; /* High-water memory dumping. */
extern bool opt_prof_final; /* Final profile dumping. */
extern bool opt_prof_leak; /* Dump leak summary at exit. */
extern bool opt_prof_leak_error; /* Exit with error code if memory leaked */
extern bool opt_prof_accum; /* Report cumulative bytes. */
extern bool opt_prof_log; /* Turn logging on at boot. */
extern char opt_prof_prefix[
/* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF
PATH_MAX +
#endif
1];
extern bool opt_prof_unbias;
/* Include pid namespace in profile file names. */
extern bool opt_prof_pid_namespace;
/* For recording recent allocations */
extern ssize_t opt_prof_recent_alloc_max;
/* Whether to use thread name provided by the system or by mallctl. */
extern bool opt_prof_sys_thread_name;
/* Whether to record per size class counts and request size totals. */
extern bool opt_prof_stats;
/* Accessed via prof_active_[gs]et{_unlocked,}(). */
extern bool prof_active_state;
/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */
extern bool prof_gdump_val;
/* Profile dump interval, measured in bytes allocated. */
extern uint64_t prof_interval;
/*
* Initialized as opt_lg_prof_sample, and potentially modified during profiling
* resets.
*/
extern size_t lg_prof_sample;
extern bool prof_booted;
void prof_backtrace_hook_set(prof_backtrace_hook_t hook);
prof_backtrace_hook_t prof_backtrace_hook_get(void);
void prof_dump_hook_set(prof_dump_hook_t hook);
prof_dump_hook_t prof_dump_hook_get(void);
void prof_sample_hook_set(prof_sample_hook_t hook);
prof_sample_hook_t prof_sample_hook_get(void);
void prof_sample_free_hook_set(prof_sample_free_hook_t hook);
prof_sample_free_hook_t prof_sample_free_hook_get(void);
/* Functions only accessed in prof_inlines.h */
prof_tdata_t *prof_tdata_init(tsd_t *tsd);
prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx);
void prof_malloc_sample_object(
tsd_t *tsd, const void *ptr, size_t size, size_t usize, prof_tctx_t *tctx);
void prof_free_sampled_object(
tsd_t *tsd, const void *ptr, size_t usize, prof_info_t *prof_info);
prof_tctx_t *prof_tctx_create(tsd_t *tsd);
void prof_idump(tsdn_t *tsdn);
bool prof_mdump(tsd_t *tsd, const char *filename);
void prof_gdump(tsdn_t *tsdn);
void prof_tdata_cleanup(tsd_t *tsd);
bool prof_active_get(tsdn_t *tsdn);
bool prof_active_set(tsdn_t *tsdn, bool active);
const char *prof_thread_name_get(tsd_t *tsd);
int prof_thread_name_set(tsd_t *tsd, const char *thread_name);
bool prof_thread_active_get(tsd_t *tsd);
bool prof_thread_active_set(tsd_t *tsd, bool active);
bool prof_thread_active_init_get(tsdn_t *tsdn);
bool prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
bool prof_gdump_get(tsdn_t *tsdn);
bool prof_gdump_set(tsdn_t *tsdn, bool active);
void prof_boot0(void);
void prof_boot1(void);
bool prof_boot2(tsd_t *tsd, base_t *base);
void prof_prefork0(tsdn_t *tsdn);
void prof_prefork1(tsdn_t *tsdn);
void prof_postfork_parent(tsdn_t *tsdn);
void prof_postfork_child(tsdn_t *tsdn);
uint64_t prof_sample_new_event_wait(tsd_t *tsd);
uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd);
/*
* The lookahead functionality facilitates events to be able to lookahead, i.e.
* without touching the event counters, to determine whether an event would be
* triggered. The event counters are not advanced until the end of the
* allocation / deallocation calls, so the lookahead can be useful if some
* preparation work for some event must be done early in the allocation /
* deallocation calls.
*
* Currently only the profiling sampling event needs the lookahead
* functionality, so we don't yet define general purpose lookahead functions.
*
* Surplus is a terminology referring to the amount of bytes beyond what's
* needed for triggering an event, which can be a useful quantity to have in
* general when lookahead is being called.
*
* This function returns true if allocation of usize would go above the next
* trigger for prof event, and false otherwise.
* If function returns true surplus will contain number of bytes beyond that
* trigger.
*/
JEMALLOC_ALWAYS_INLINE bool
te_prof_sample_event_lookahead_surplus(
tsd_t *tsd, size_t usize, size_t *surplus) {
if (surplus != NULL) {
/*
* This is a dead store: the surplus will be overwritten before
* any read. The initialization suppresses compiler warnings.
* Meanwhile, using SIZE_MAX to initialize is good for
* debugging purpose, because a valid surplus value is strictly
* less than usize, which is at most SIZE_MAX.
*/
*surplus = SIZE_MAX;
}
if (unlikely(!tsd_nominal(tsd) || tsd_reentrancy_level_get(tsd) > 0)) {
return false;
}
/* The subtraction is intentionally susceptible to underflow. */
uint64_t accumbytes = tsd_thread_allocated_get(tsd) + usize
- tsd_thread_allocated_last_event_get(tsd);
uint64_t sample_wait = tsd_prof_sample_event_wait_get(tsd);
if (accumbytes < sample_wait) {
return false;
}
assert(accumbytes - sample_wait < (uint64_t)usize);
if (surplus != NULL) {
*surplus = (size_t)(accumbytes - sample_wait);
}
return true;
}
JEMALLOC_ALWAYS_INLINE bool
te_prof_sample_event_lookahead(tsd_t *tsd, size_t usize) {
return te_prof_sample_event_lookahead_surplus(tsd, usize, NULL);
}
extern te_base_cb_t prof_sample_te_handler;
#endif /* JEMALLOC_INTERNAL_PROF_EXTERNS_H */

5
include/jemalloc/internal/prof_inlines.h
View File

@@ -2,9 +2,10 @@
#define JEMALLOC_INTERNAL_PROF_INLINES_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena_inlines.h"
#include "jemalloc/internal/arena_inlines_b.h"
#include "jemalloc/internal/jemalloc_internal_inlines_c.h"
#include "jemalloc/internal/prof.h"
#include "jemalloc/internal/prof_externs.h"
#include "jemalloc/internal/prof_structs.h"
#include "jemalloc/internal/safety_check.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/thread_event.h"

222
include/jemalloc/internal/prof_structs.h Normal file
View File

@@ -0,0 +1,222 @@
#ifndef JEMALLOC_INTERNAL_PROF_STRUCTS_H
#define JEMALLOC_INTERNAL_PROF_STRUCTS_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/ckh.h"
#include "jemalloc/internal/edata.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/rb.h"
struct prof_bt_s {
/* Backtrace, stored as len program counters. */
void **vec;
unsigned len;
};
#ifdef JEMALLOC_PROF_LIBGCC
/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */
typedef struct {
void **vec;
unsigned *len;
unsigned max;
} prof_unwind_data_t;
#endif
struct prof_cnt_s {
/* Profiling counters. */
uint64_t curobjs;
uint64_t curobjs_shifted_unbiased;
uint64_t curbytes;
uint64_t curbytes_unbiased;
uint64_t accumobjs;
uint64_t accumobjs_shifted_unbiased;
uint64_t accumbytes;
uint64_t accumbytes_unbiased;
};
typedef enum {
prof_tctx_state_initializing,
prof_tctx_state_nominal,
prof_tctx_state_dumping,
prof_tctx_state_purgatory /* Dumper must finish destroying. */
} prof_tctx_state_t;
struct prof_tctx_s {
/* Thread data for thread that performed the allocation. */
prof_tdata_t *tdata;
/*
* Copy of tdata->thr_{uid,discrim}, necessary because tdata may be
* defunct during teardown.
*/
uint64_t thr_uid;
uint64_t thr_discrim;
/*
* Reference count of how many times this tctx object is referenced in
* recent allocation / deallocation records, protected by tdata->lock.
*/
uint64_t recent_count;
/* Profiling counters, protected by tdata->lock. */
prof_cnt_t cnts;
/* Associated global context. */
prof_gctx_t *gctx;
/*
* UID that distinguishes multiple tctx's created by the same thread,
* but coexisting in gctx->tctxs. There are two ways that such
* coexistence can occur:
* - A dumper thread can cause a tctx to be retained in the purgatory
* state.
* - Although a single "producer" thread must create all tctx's which
* share the same thr_uid, multiple "consumers" can each concurrently
* execute portions of prof_tctx_destroy(). prof_tctx_destroy() only
* gets called once each time cnts.cur{objs,bytes} drop to 0, but this
* threshold can be hit again before the first consumer finishes
* executing prof_tctx_destroy().
*/
uint64_t tctx_uid;
/* Linkage into gctx's tctxs. */
rb_node(prof_tctx_t) tctx_link;
/*
* True during prof_alloc_prep()..prof_malloc_sample_object(), prevents
* sample vs destroy race.
*/
bool prepared;
/* Current dump-related state, protected by gctx->lock. */
prof_tctx_state_t state;
/*
* Copy of cnts snapshotted during early dump phase, protected by
* dump_mtx.
*/
prof_cnt_t dump_cnts;
};
typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;
struct prof_info_s {
/* Time when the allocation was made. */
nstime_t alloc_time;
/* Points to the prof_tctx_t corresponding to the allocation. */
prof_tctx_t *alloc_tctx;
/* Allocation request size. */
size_t alloc_size;
};
struct prof_gctx_s {
/* Protects nlimbo, cnt_summed, and tctxs. */
malloc_mutex_t *lock;
/*
* Number of threads that currently cause this gctx to be in a state of
* limbo due to one of:
* - Initializing this gctx.
* - Initializing per thread counters associated with this gctx.
* - Preparing to destroy this gctx.
* - Dumping a heap profile that includes this gctx.
* nlimbo must be 1 (single destroyer) in order to safely destroy the
* gctx.
*/
unsigned nlimbo;
/*
* Tree of profile counters, one for each thread that has allocated in
* this context.
*/
prof_tctx_tree_t tctxs;
/* Linkage for tree of contexts to be dumped. */
rb_node(prof_gctx_t) dump_link;
/* Temporary storage for summation during dump. */
prof_cnt_t cnt_summed;
/* Associated backtrace. */
prof_bt_t bt;
/* Backtrace vector, variable size, referred to by bt. */
void *vec[1];
};
typedef rb_tree(prof_gctx_t) prof_gctx_tree_t;
struct prof_tdata_s {
malloc_mutex_t *lock;
/* Monotonically increasing unique thread identifier. */
uint64_t thr_uid;
/*
* Monotonically increasing discriminator among tdata structures
* associated with the same thr_uid.
*/
uint64_t thr_discrim;
rb_node(prof_tdata_t) tdata_link;
/*
* Counter used to initialize prof_tctx_t's tctx_uid. No locking is
* necessary when incrementing this field, because only one thread ever
* does so.
*/
uint64_t tctx_uid_next;
/*
* Hash of (prof_bt_t *)-->(prof_tctx_t *). Each thread tracks
* backtraces for which it has non-zero allocation/deallocation counters
* associated with thread-specific prof_tctx_t objects. Other threads
* may write to prof_tctx_t contents when freeing associated objects.
*/
ckh_t bt2tctx;
/* Included in heap profile dumps if has content. */
char thread_name[PROF_THREAD_NAME_MAX_LEN];
/* State used to avoid dumping while operating on prof internals. */
bool enq;
bool enq_idump;
bool enq_gdump;
/*
* Set to true during an early dump phase for tdata's which are
* currently being dumped. New threads' tdata's have this initialized
* to false so that they aren't accidentally included in later dump
* phases.
*/
bool dumping;
/*
* True if profiling is active for this tdata's thread
* (thread.prof.active mallctl).
*/
bool active;
bool attached;
bool expired;
/* Temporary storage for summation during dump. */
prof_cnt_t cnt_summed;
/* Backtrace vector, used for calls to prof_backtrace(). */
void **vec;
};
typedef rb_tree(prof_tdata_t) prof_tdata_tree_t;
struct prof_recent_s {
nstime_t alloc_time;
nstime_t dalloc_time;
ql_elm(prof_recent_t) link;
size_t size;
size_t usize;
atomic_p_t alloc_edata; /* NULL means allocation has been freed. */
prof_tctx_t *alloc_tctx;
prof_tctx_t *dalloc_tctx;
};
#endif /* JEMALLOC_INTERNAL_PROF_STRUCTS_H */

1
include/jemalloc/internal/prof_sys.h
View File

@@ -4,7 +4,6 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/base.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/prof.h"
extern malloc_mutex_t prof_dump_filename_mtx;
extern base_t *prof_base;

94
include/jemalloc/internal/prof_types.h Normal file
View File

@@ -0,0 +1,94 @@
#ifndef JEMALLOC_INTERNAL_PROF_TYPES_H
#define JEMALLOC_INTERNAL_PROF_TYPES_H
typedef struct prof_bt_s prof_bt_t;
typedef struct prof_cnt_s prof_cnt_t;
typedef struct prof_tctx_s prof_tctx_t;
typedef struct prof_info_s prof_info_t;
typedef struct prof_gctx_s prof_gctx_t;
typedef struct prof_tdata_s prof_tdata_t;
typedef struct prof_recent_s prof_recent_t;
/* Option defaults. */
#ifdef JEMALLOC_PROF
# define PROF_PREFIX_DEFAULT "jeprof"
#else
# define PROF_PREFIX_DEFAULT ""
#endif
#define LG_PROF_SAMPLE_DEFAULT 19
#define LG_PROF_INTERVAL_DEFAULT -1
/*
* Hard limit on stack backtrace depth. The version of prof_backtrace() that
* is based on __builtin_return_address() necessarily has a hard-coded number
* of backtrace frame handlers, and should be kept in sync with this setting.
*/
#ifdef JEMALLOC_PROF_GCC
# define PROF_BT_MAX_LIMIT 256
#else
# define PROF_BT_MAX_LIMIT UINT_MAX
#endif
#define PROF_BT_MAX_DEFAULT 128
/* Initial hash table size. */
#define PROF_CKH_MINITEMS 64
/* Size of memory buffer to use when writing dump files. */
#ifndef JEMALLOC_PROF
/* Minimize memory bloat for non-prof builds. */
# define PROF_DUMP_BUFSIZE 1
#elif defined(JEMALLOC_DEBUG)
/* Use a small buffer size in debug build, mainly to facilitate testing. */
# define PROF_DUMP_BUFSIZE 16
#else
# define PROF_DUMP_BUFSIZE 65536
#endif
/* Size of size class related tables */
#ifdef JEMALLOC_PROF
# define PROF_SC_NSIZES SC_NSIZES
#else
/* Minimize memory bloat for non-prof builds. */
# define PROF_SC_NSIZES 1
#endif
/* Size of stack-allocated buffer used by prof_printf(). */
#define PROF_PRINTF_BUFSIZE 128
/*
* Number of mutexes shared among all gctx's. No space is allocated for these
* unless profiling is enabled, so it's okay to over-provision.
*/
#define PROF_NCTX_LOCKS 1024
/*
* Number of mutexes shared among all tdata's. No space is allocated for these
* unless profiling is enabled, so it's okay to over-provision.
*/
#define PROF_NTDATA_LOCKS 256
/* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF
# define PROF_DUMP_FILENAME_LEN (PATH_MAX + 1)
#else
# define PROF_DUMP_FILENAME_LEN 1
#endif
/* Default number of recent allocations to record. */
#define PROF_RECENT_ALLOC_MAX_DEFAULT 0
/* Thread name storage size limit. */
#define PROF_THREAD_NAME_MAX_LEN 16
/*
* Minimum required alignment for sampled allocations. Over-aligning sampled
* allocations allows us to quickly identify them on the dalloc path without
* resorting to metadata lookup.
*/
#define PROF_SAMPLE_ALIGNMENT PAGE
#define PROF_SAMPLE_ALIGNMENT_MASK PAGE_MASK
/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
#define PROF_TCTX_SENTINEL ((prof_tctx_t *)((uintptr_t)1U))
#endif /* JEMALLOC_INTERNAL_PROF_TYPES_H */

6
include/jemalloc/internal/psset.h
View File

@@ -134,17 +134,17 @@ void psset_insert(psset_t *psset, hpdata_t *ps);
void psset_remove(psset_t *psset, hpdata_t *ps);
static inline size_t
psset_npageslabs(const psset_t *psset) {
psset_npageslabs(psset_t *psset) {
return psset->stats.merged.npageslabs;
}
static inline size_t
psset_nactive(const psset_t *psset) {
psset_nactive(psset_t *psset) {
return psset->stats.merged.nactive;
}
static inline size_t
psset_ndirty(const psset_t *psset) {
psset_ndirty(psset_t *psset) {
return psset->stats.merged.ndirty;
}

5
include/jemalloc/internal/safety_check.h
View File

@@ -11,6 +11,11 @@ void safety_check_fail_sized_dealloc(
bool current_dealloc, const void *ptr, size_t true_size, size_t input_size);
void safety_check_fail(const char *format, ...);
typedef void (*safety_check_abort_hook_t)(const char *message);
/* Can set to NULL for a default. */
void safety_check_set_abort(safety_check_abort_hook_t abort_fn);
#define REDZONE_SIZE ((size_t)32)
#define REDZONE_FILL_VALUE 0xBC

17
include/jemalloc/internal/sc.h
View File

@@ -268,17 +268,12 @@
#define SC_LARGE_MINCLASS ((size_t)1ULL << (LG_PAGE + SC_LG_NGROUP))
#define SC_LG_LARGE_MINCLASS (LG_PAGE + SC_LG_NGROUP)
/*
* The largest size class supported. Spell this out directly to avoid
* expanding subtractive arithmetic at every use site.
*/
#if LG_SIZEOF_PTR == 3
# define SC_LARGE_MAXCLASS 0x7000000000000000ULL
#elif LG_SIZEOF_PTR == 2
# define SC_LARGE_MAXCLASS 0x70000000ULL
#else
# error "Unsupported pointer size"
#endif
/* Internal; only used for the definition of SC_LARGE_MAXCLASS. */
#define SC_MAX_BASE ((size_t)1 << (SC_PTR_BITS - 2))
#define SC_MAX_DELTA ((size_t)1 << (SC_PTR_BITS - 2 - SC_LG_NGROUP))
/* The largest size class supported. */
#define SC_LARGE_MAXCLASS (SC_MAX_BASE + (SC_NGROUP - 1) * SC_MAX_DELTA)
/* Maximum number of regions in one slab. */
#ifndef CONFIG_LG_SLAB_MAXREGS

22
include/jemalloc/internal/sec.h
View File

@@ -4,10 +4,9 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/base.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/edata.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/pai.h"
#include "jemalloc/internal/sec_opts.h"
#include "jemalloc/internal/tsd_types.h"
/*
* Small extent cache.
@@ -97,25 +96,6 @@ sec_size_supported(sec_t *sec, size_t size) {
return sec_is_used(sec) && size <= sec->opts.max_alloc;
}
/* Min number of extents we will allocate when expanding the SEC. */
#define SEC_MIN_NALLOCS 2
/* Max number of extents we will allocate out of a single huge page. */
#define SEC_MAX_NALLOCS 8
/* Attempt to fill the SEC up to max_bytes / SEC_MAX_BYTES_DIV */
#define SEC_MAX_BYTES_DIV 4
/*
* Calculate the min and max number of extents we will try to allocate
* when expanding the SEC. We will attempt to allocate at least min
* extents and up to max extents depending on whether we can allocate
* them out of a huge page we have already allocated out of. Both
* min and max should the in the range [1, SEC_MAX_NALLOCS].
*/
void sec_calc_nallocs_for_size(
sec_t *sec, size_t size, size_t *min_nallocs, size_t *max_nallocs);
/* If sec does not have extent available, it will return NULL. */
edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size);
void sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,

9
include/jemalloc/internal/sec_opts.h
View File

@@ -27,9 +27,16 @@ struct sec_opts_s {
* until we are 1/4 below max_bytes.
*/
size_t max_bytes;
/*
* When we can't satisfy an allocation out of the SEC because there are
* no available ones cached, allocator will allocate a batch with extra
* batch_fill_extra extents of the same size.
*/
size_t batch_fill_extra;
};
#define SEC_OPTS_NSHARDS_DEFAULT 2
#define SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT 3
#define SEC_OPTS_MAX_ALLOC_DEFAULT ((32 * 1024) < PAGE ? PAGE : (32 * 1024))
#define SEC_OPTS_MAX_BYTES_DEFAULT \
((256 * 1024) < (4 * SEC_OPTS_MAX_ALLOC_DEFAULT) \
@@ -38,6 +45,6 @@ struct sec_opts_s {
#define SEC_OPTS_DEFAULT \
{SEC_OPTS_NSHARDS_DEFAULT, SEC_OPTS_MAX_ALLOC_DEFAULT, \
SEC_OPTS_MAX_BYTES_DEFAULT}
SEC_OPTS_MAX_BYTES_DEFAULT, SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT}
#endif /* JEMALLOC_INTERNAL_SEC_OPTS_H */

58
include/jemalloc/internal/seq.h Normal file
View File

@@ -0,0 +1,58 @@
#ifndef JEMALLOC_INTERNAL_SEQ_H
#define JEMALLOC_INTERNAL_SEQ_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/atomic.h"
/*
* A simple seqlock implementation.
*/
/* clang-format off */
#define seq_define(type, short_type) \
typedef struct { \
atomic_zu_t seq; \
atomic_zu_t data[ \
(sizeof(type) + sizeof(size_t) - 1) / sizeof(size_t)]; \
} seq_##short_type##_t; \
\
/* \
* No internal synchronization -- the caller must ensure that there's \
* only a single writer at a time. \
*/ \
static inline void \
seq_store_##short_type(seq_##short_type##_t *dst, type *src) { \
size_t buf[sizeof(dst->data) / sizeof(size_t)]; \
buf[sizeof(buf) / sizeof(size_t) - 1] = 0; \
memcpy(buf, src, sizeof(type)); \
size_t old_seq = atomic_load_zu(&dst->seq, ATOMIC_RELAXED); \
atomic_store_zu(&dst->seq, old_seq + 1, ATOMIC_RELAXED); \
atomic_fence(ATOMIC_RELEASE); \
for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) { \
atomic_store_zu(&dst->data[i], buf[i], ATOMIC_RELAXED); \
} \
atomic_store_zu(&dst->seq, old_seq + 2, ATOMIC_RELEASE); \
} \
\
/* Returns whether or not the read was consistent. */ \
static inline bool \
seq_try_load_##short_type(type *dst, seq_##short_type##_t *src) { \
size_t buf[sizeof(src->data) / sizeof(size_t)]; \
size_t seq1 = atomic_load_zu(&src->seq, ATOMIC_ACQUIRE); \
if (seq1 % 2 != 0) { \
return false; \
} \
for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) { \
buf[i] = atomic_load_zu(&src->data[i], ATOMIC_RELAXED); \
} \
atomic_fence(ATOMIC_ACQUIRE); \
size_t seq2 = atomic_load_zu(&src->seq, ATOMIC_RELAXED); \
if (seq1 != seq2) { \
return false; \
} \
memcpy(dst, buf, sizeof(type)); \
return true; \
}
/* clang-format on */
#endif /* JEMALLOC_INTERNAL_SEQ_H */

3
include/jemalloc/internal/stats.h
View File

@@ -43,9 +43,6 @@ extern char opt_stats_interval_opts[stats_print_tot_num_options + 1];
#define STATS_INTERVAL_ACCUM_LG_BATCH_SIZE 6
#define STATS_INTERVAL_ACCUM_BATCH_MAX (4 << 20)
/* Per thread batch accum size for stats_interval; read by thread event. */
extern uint64_t stats_interval_accum_batch;
/* Only accessed by thread event. */
extern te_base_cb_t stats_interval_te_handler;

20
include/jemalloc/internal/sz.h
View File

@@ -3,7 +3,6 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/jemalloc_internal_externs.h"
#include "jemalloc/internal/pages.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/util.h"
@@ -315,20 +314,12 @@ sz_size2index_usize_fastpath(size_t size, szind_t *ind, size_t *usize) {
JEMALLOC_ALWAYS_INLINE size_t
sz_s2u_compute_using_delta(size_t size) {
if (unlikely(size > SC_LARGE_MAXCLASS)) {
return 0;
}
size_t x = lg_floor((size << 1) - 1);
size_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1)
? LG_QUANTUM
: x - SC_LG_NGROUP - 1;
size_t delta = ZU(1) << lg_delta;
size_t delta_mask = delta - 1;
if (unlikely(size > SIZE_T_MAX - delta_mask)) {
return 0;
}
size_t usize = (size + delta_mask) & ~delta_mask;
return usize;
}
@@ -394,11 +385,7 @@ JEMALLOC_ALWAYS_INLINE size_t
sz_sa2u(size_t size, size_t alignment) {
size_t usize;
if (unlikely(alignment == 0)) {
return 0;
}
size_t alignment_mask = alignment - 1;
assert((alignment_mask & alignment) == 0);
assert(alignment != 0 && ((alignment - 1) & alignment) == 0);
/* Try for a small size class. */
if (size <= SC_SMALL_MAXCLASS && alignment <= PAGE) {
@@ -416,10 +403,7 @@ sz_sa2u(size_t size, size_t alignment) {
* 144 | 10100000 | 32
* 192 | 11000000 | 64
*/
if (unlikely(size > SIZE_T_MAX - alignment_mask)) {
return 0;
}
usize = sz_s2u((size + alignment_mask) & ~alignment_mask);
usize = sz_s2u(ALIGNMENT_CEILING(size, alignment));
if (usize < SC_LARGE_MINCLASS) {
return usize;
}

199
include/jemalloc/internal/tcache.h
View File

@@ -1,199 +0,0 @@
#ifndef JEMALLOC_INTERNAL_TCACHE_H
#define JEMALLOC_INTERNAL_TCACHE_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/cache_bin.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/thread_event_registry.h"
#include "jemalloc/internal/ticker.h"
/* Forward decls; only used as pointer types below. */
typedef struct arena_s arena_t;
typedef struct base_s base_t;
/******************************************************************************/
/* TYPES */
/******************************************************************************/
typedef struct tcache_slow_s tcache_slow_t;
typedef struct tcache_s tcache_t;
typedef struct tcaches_s tcaches_t;
/* Used in TSD static initializer only. Real init in tsd_tcache_data_init(). */
#define TCACHE_ZERO_INITIALIZER \
{ 0 }
#define TCACHE_SLOW_ZERO_INITIALIZER \
{ \
{ 0 } \
}
/* Used in TSD static initializer only. Will be initialized to opt_tcache. */
#define TCACHE_ENABLED_ZERO_INITIALIZER false
/* Used for explicit tcache only. Means flushed but not destroyed. */
/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
#define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1)
#define TCACHE_LG_MAXCLASS_LIMIT LG_USIZE_GROW_SLOW_THRESHOLD
#define TCACHE_MAXCLASS_LIMIT ((size_t)1 << TCACHE_LG_MAXCLASS_LIMIT)
#define TCACHE_NBINS_MAX \
(SC_NBINS \
+ SC_NGROUP * (TCACHE_LG_MAXCLASS_LIMIT - SC_LG_LARGE_MINCLASS) \
+ 1)
#define TCACHE_GC_NEIGHBOR_LIMIT ((uintptr_t)1 << 21) /* 2M */
#define TCACHE_GC_INTERVAL_NS ((uint64_t)10 * KQU(1000000)) /* 10ms */
#define TCACHE_GC_SMALL_NBINS_MAX ((SC_NBINS > 8) ? (SC_NBINS >> 3) : 1)
#define TCACHE_GC_LARGE_NBINS_MAX 1
/******************************************************************************/
/* STRUCTS */
/******************************************************************************/
/*
* The tcache state is split into the slow and hot path data. Each has a
* pointer to the other, and the data always comes in pairs. The layout of each
* of them varies in practice; tcache_slow lives in the TSD for the automatic
* tcache, and as part of a dynamic allocation for manual allocations. Keeping
* a pointer to tcache_slow lets us treat these cases uniformly, rather than
* splitting up the tcache [de]allocation code into those paths called with the
* TSD tcache and those called with a manual tcache.
*/
struct tcache_slow_s {
/*
* The descriptor lets the arena find our cache bins without seeing the
* tcache definition. This enables arenas to aggregate stats across
* tcaches without having a tcache dependency.
*/
cache_bin_array_descriptor_t cache_bin_array_descriptor;
/* The arena this tcache is associated with. */
arena_t *arena;
/* The number of bins activated in the tcache. */
unsigned tcache_nbins;
/* Last time GC has been performed. */
nstime_t last_gc_time;
/* Next bin to GC. */
szind_t next_gc_bin;
szind_t next_gc_bin_small;
szind_t next_gc_bin_large;
/* For small bins, help determine how many items to fill at a time. */
cache_bin_fill_ctl_t bin_fill_ctl_do_not_access_directly[SC_NBINS];
/* For small bins, whether has been refilled since last GC. */
bool bin_refilled[SC_NBINS];
/*
* For small bins, the number of items we can pretend to flush before
* actually flushing.
*/
uint8_t bin_flush_delay_items[SC_NBINS];
/*
* The start of the allocation containing the dynamic allocation for
* either the cache bins alone, or the cache bin memory as well as this
* tcache_slow_t and its associated tcache_t.
*/
void *dyn_alloc;
/* The associated bins. */
tcache_t *tcache;
};
struct tcache_s {
tcache_slow_t *tcache_slow;
cache_bin_t bins[TCACHE_NBINS_MAX];
};
/* Linkage for list of available (previously used) explicit tcache IDs. */
struct tcaches_s {
union {
tcache_t *tcache;
tcaches_t *next;
};
};
/******************************************************************************/
/* EXTERNS */
/******************************************************************************/
extern bool opt_tcache;
extern size_t opt_tcache_max;
extern ssize_t opt_lg_tcache_nslots_mul;
extern unsigned opt_tcache_nslots_small_min;
extern unsigned opt_tcache_nslots_small_max;
extern unsigned opt_tcache_nslots_large;
extern ssize_t opt_lg_tcache_shift;
extern size_t opt_tcache_gc_incr_bytes;
extern size_t opt_tcache_gc_delay_bytes;
extern unsigned opt_lg_tcache_flush_small_div;
extern unsigned opt_lg_tcache_flush_large_div;
/*
* Number of tcache bins. There are SC_NBINS small-object bins, plus 0 or more
* large-object bins. This is only used during threads initialization and
* changing it will not reflect on initialized threads as expected. Thus,
* it should not be changed on the fly. To change the number of tcache bins
* in use, refer to tcache_nbins of each tcache.
*/
extern unsigned global_do_not_change_tcache_nbins;
/*
* Maximum cached size class. Same as above, this is only used during threads
* initialization and should not be changed. To change the maximum cached size
* class, refer to tcache_max of each tcache.
*/
extern size_t global_do_not_change_tcache_maxclass;
/*
* Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
* usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are
* completely disjoint from this data structure. tcaches starts off as a sparse
* array, so it has no physical memory footprint until individual pages are
* touched. This allows the entire array to be allocated the first time an
* explicit tcache is created without a disproportionate impact on memory usage.
*/
extern tcaches_t *tcaches;
size_t tcache_salloc(tsdn_t *tsdn, const void *ptr);
void *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, bool *tcache_success);
void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, unsigned rem);
void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, unsigned rem);
void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, bool is_small);
bool tcache_bin_info_default_init(
const char *bin_settings_segment_cur, size_t len_left);
bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len);
bool tcache_bin_ncached_max_read(
tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max);
void tcache_arena_reassociate(
tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
tcache_t *tcache_create_explicit(tsd_t *tsd);
bool thread_tcache_max_set(tsd_t *tsd, size_t tcache_max);
void tcache_cleanup(tsd_t *tsd);
bool tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
void tcaches_flush(tsd_t *tsd, unsigned ind);
void tcaches_destroy(tsd_t *tsd, unsigned ind);
bool tcache_boot(tsdn_t *tsdn, base_t *base);
void tcache_arena_associate(
tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
cache_bin_array_descriptor_t *tcache_postfork_arena_descriptor(
tsdn_t *tsdn, arena_t *arena);
void tcache_prefork(tsdn_t *tsdn);
void tcache_postfork_parent(tsdn_t *tsdn);
void tcache_postfork_child(tsdn_t *tsdn);
void tcache_flush(tsd_t *tsd);
bool tsd_tcache_enabled_data_init(tsd_t *tsd);
void tcache_enabled_set(tsd_t *tsd, bool enabled);
extern void *(*JET_MUTABLE tcache_stack_alloc)(tsdn_t *tsdn, size_t size,
size_t alignment);
void tcache_assert_initialized(tcache_t *tcache);
extern te_base_cb_t tcache_gc_te_handler;
#endif /* JEMALLOC_INTERNAL_TCACHE_H */

90
include/jemalloc/internal/tcache_externs.h Normal file
View File

@@ -0,0 +1,90 @@
#ifndef JEMALLOC_INTERNAL_TCACHE_EXTERNS_H
#define JEMALLOC_INTERNAL_TCACHE_EXTERNS_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/base.h"
#include "jemalloc/internal/cache_bin.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/tcache_types.h"
#include "jemalloc/internal/thread_event_registry.h"
extern bool opt_tcache;
extern size_t opt_tcache_max;
extern ssize_t opt_lg_tcache_nslots_mul;
extern unsigned opt_tcache_nslots_small_min;
extern unsigned opt_tcache_nslots_small_max;
extern unsigned opt_tcache_nslots_large;
extern ssize_t opt_lg_tcache_shift;
extern size_t opt_tcache_gc_incr_bytes;
extern size_t opt_tcache_gc_delay_bytes;
extern unsigned opt_lg_tcache_flush_small_div;
extern unsigned opt_lg_tcache_flush_large_div;
/*
* Number of tcache bins. There are SC_NBINS small-object bins, plus 0 or more
* large-object bins. This is only used during threads initialization and
* changing it will not reflect on initialized threads as expected. Thus,
* it should not be changed on the fly. To change the number of tcache bins
* in use, refer to tcache_nbins of each tcache.
*/
extern unsigned global_do_not_change_tcache_nbins;
/*
* Maximum cached size class. Same as above, this is only used during threads
* initialization and should not be changed. To change the maximum cached size
* class, refer to tcache_max of each tcache.
*/
extern size_t global_do_not_change_tcache_maxclass;
/*
* Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
* usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are
* completely disjoint from this data structure. tcaches starts off as a sparse
* array, so it has no physical memory footprint until individual pages are
* touched. This allows the entire array to be allocated the first time an
* explicit tcache is created without a disproportionate impact on memory usage.
*/
extern tcaches_t *tcaches;
size_t tcache_salloc(tsdn_t *tsdn, const void *ptr);
void *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, bool *tcache_success);
void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, unsigned rem);
void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, unsigned rem);
void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, bool is_small);
bool tcache_bin_info_default_init(
const char *bin_settings_segment_cur, size_t len_left);
bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len);
bool tcache_bin_ncached_max_read(
tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max);
void tcache_arena_reassociate(
tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena);
tcache_t *tcache_create_explicit(tsd_t *tsd);
bool thread_tcache_max_set(tsd_t *tsd, size_t tcache_max);
void tcache_cleanup(tsd_t *tsd);
void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
bool tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
void tcaches_flush(tsd_t *tsd, unsigned ind);
void tcaches_destroy(tsd_t *tsd, unsigned ind);
bool tcache_boot(tsdn_t *tsdn, base_t *base);
void tcache_arena_associate(
tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena);
void tcache_prefork(tsdn_t *tsdn);
void tcache_postfork_parent(tsdn_t *tsdn);
void tcache_postfork_child(tsdn_t *tsdn);
void tcache_flush(tsd_t *tsd);
bool tsd_tcache_enabled_data_init(tsd_t *tsd);
void tcache_enabled_set(tsd_t *tsd, bool enabled);
extern void *(*JET_MUTABLE tcache_stack_alloc)(tsdn_t *tsdn, size_t size,
size_t alignment);
void tcache_assert_initialized(tcache_t *tcache);
extern te_base_cb_t tcache_gc_te_handler;
#endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */

19
include/jemalloc/internal/tcache_inlines.h
View File

@@ -2,15 +2,15 @@
#define JEMALLOC_INTERNAL_TCACHE_INLINES_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/arena_externs.h"
#include "jemalloc/internal/bin.h"
#include "jemalloc/internal/arena_inlines.h"
#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/large.h"
#include "jemalloc/internal/large_externs.h"
#include "jemalloc/internal/san.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/tcache_externs.h"
#include "jemalloc/internal/util.h"
static inline bool
@@ -19,7 +19,7 @@ tcache_enabled_get(tsd_t *tsd) {
}
static inline unsigned
tcache_nbins_get(const tcache_slow_t *tcache_slow) {
tcache_nbins_get(tcache_slow_t *tcache_slow) {
assert(tcache_slow != NULL);
unsigned nbins = tcache_slow->tcache_nbins;
assert(nbins <= TCACHE_NBINS_MAX);
@@ -27,7 +27,7 @@ tcache_nbins_get(const tcache_slow_t *tcache_slow) {
}
static inline size_t
tcache_max_get(const tcache_slow_t *tcache_slow) {
tcache_max_get(tcache_slow_t *tcache_slow) {
assert(tcache_slow != NULL);
size_t tcache_max = sz_index2size(tcache_nbins_get(tcache_slow) - 1);
assert(tcache_max <= TCACHE_MAXCLASS_LIMIT);
@@ -86,13 +86,6 @@ tcache_bin_disabled(szind_t ind, cache_bin_t *bin, tcache_slow_t *tcache_slow) {
return disabled;
}
JEMALLOC_ALWAYS_INLINE bool
tcache_can_cache_large(tcache_t *tcache, szind_t ind) {
return ind < tcache_nbins_get(tcache->tcache_slow)
&& !tcache_bin_disabled(ind, &tcache->bins[ind],
tcache->tcache_slow);
}
JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
szind_t binind, bool zero, bool slow_path) {

75
include/jemalloc/internal/tcache_structs.h Normal file
View File

@@ -0,0 +1,75 @@
#ifndef JEMALLOC_INTERNAL_TCACHE_STRUCTS_H
#define JEMALLOC_INTERNAL_TCACHE_STRUCTS_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/cache_bin.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/tcache_types.h"
#include "jemalloc/internal/ticker.h"
/*
* The tcache state is split into the slow and hot path data. Each has a
* pointer to the other, and the data always comes in pairs. The layout of each
* of them varies in practice; tcache_slow lives in the TSD for the automatic
* tcache, and as part of a dynamic allocation for manual allocations. Keeping
* a pointer to tcache_slow lets us treat these cases uniformly, rather than
* splitting up the tcache [de]allocation code into those paths called with the
* TSD tcache and those called with a manual tcache.
*/
struct tcache_slow_s {
/* Lets us track all the tcaches in an arena. */
ql_elm(tcache_slow_t) link;
/*
* The descriptor lets the arena find our cache bins without seeing the
* tcache definition. This enables arenas to aggregate stats across
* tcaches without having a tcache dependency.
*/
cache_bin_array_descriptor_t cache_bin_array_descriptor;
/* The arena this tcache is associated with. */
arena_t *arena;
/* The number of bins activated in the tcache. */
unsigned tcache_nbins;
/* Last time GC has been performed. */
nstime_t last_gc_time;
/* Next bin to GC. */
szind_t next_gc_bin;
szind_t next_gc_bin_small;
szind_t next_gc_bin_large;
/* For small bins, help determine how many items to fill at a time. */
cache_bin_fill_ctl_t bin_fill_ctl_do_not_access_directly[SC_NBINS];
/* For small bins, whether has been refilled since last GC. */
bool bin_refilled[SC_NBINS];
/*
* For small bins, the number of items we can pretend to flush before
* actually flushing.
*/
uint8_t bin_flush_delay_items[SC_NBINS];
/*
* The start of the allocation containing the dynamic allocation for
* either the cache bins alone, or the cache bin memory as well as this
* tcache_slow_t and its associated tcache_t.
*/
void *dyn_alloc;
/* The associated bins. */
tcache_t *tcache;
};
struct tcache_s {
tcache_slow_t *tcache_slow;
cache_bin_t bins[TCACHE_NBINS_MAX];
};
/* Linkage for list of available (previously used) explicit tcache IDs. */
struct tcaches_s {
union {
tcache_t *tcache;
tcaches_t *next;
};
};
#endif /* JEMALLOC_INTERNAL_TCACHE_STRUCTS_H */

37
include/jemalloc/internal/tcache_types.h Normal file
View File

@@ -0,0 +1,37 @@
#ifndef JEMALLOC_INTERNAL_TCACHE_TYPES_H
#define JEMALLOC_INTERNAL_TCACHE_TYPES_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/sc.h"
typedef struct tcache_slow_s tcache_slow_t;
typedef struct tcache_s tcache_t;
typedef struct tcaches_s tcaches_t;
/* Used in TSD static initializer only. Real init in tsd_tcache_data_init(). */
#define TCACHE_ZERO_INITIALIZER \
{ 0 }
#define TCACHE_SLOW_ZERO_INITIALIZER \
{ \
{ 0 } \
}
/* Used in TSD static initializer only. Will be initialized to opt_tcache. */
#define TCACHE_ENABLED_ZERO_INITIALIZER false
/* Used for explicit tcache only. Means flushed but not destroyed. */
/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
#define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1)
#define TCACHE_LG_MAXCLASS_LIMIT LG_USIZE_GROW_SLOW_THRESHOLD
#define TCACHE_MAXCLASS_LIMIT ((size_t)1 << TCACHE_LG_MAXCLASS_LIMIT)
#define TCACHE_NBINS_MAX \
(SC_NBINS \
+ SC_NGROUP * (TCACHE_LG_MAXCLASS_LIMIT - SC_LG_LARGE_MINCLASS) \
+ 1)
#define TCACHE_GC_NEIGHBOR_LIMIT ((uintptr_t)1 << 21) /* 2M */
#define TCACHE_GC_INTERVAL_NS ((uint64_t)10 * KQU(1000000)) /* 10ms */
#define TCACHE_GC_SMALL_NBINS_MAX ((SC_NBINS > 8) ? (SC_NBINS >> 3) : 1)
#define TCACHE_GC_LARGE_NBINS_MAX 1
#endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */

3
include/jemalloc/internal/test_hooks.h
View File

@@ -1,9 +1,10 @@
#ifndef JEMALLOC_INTERNAL_TEST_HOOKS_H
#define JEMALLOC_INTERNAL_TEST_HOOKS_H
#include "jemalloc/internal/jemalloc_preamble.h"
extern JEMALLOC_EXPORT void (*test_hooks_arena_new_hook)(void);
extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(void);
extern JEMALLOC_EXPORT void (*test_hooks_safety_check_abort)(const char *);
#if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
# define JEMALLOC_TEST_HOOK(fn, hook) \

10
include/jemalloc/internal/tsd.h
View File

@@ -37,12 +37,13 @@ TSD_DATA_SLOWER
JEMALLOC_ALWAYS_INLINE t * \
tsd_##n##p_get(tsd_t *tsd) { \
/* \
* Read the state once, so that a transition between \
* nominal states does not confuse this assertion. \
* Because the state might change asynchronously if it's \
* nominal, we need to make sure that we only read it once. \
*/ \
uint8_t state = tsd_state_get(tsd); \
assert(state == tsd_state_nominal || \
state == tsd_state_nominal_slow || \
state == tsd_state_nominal_recompute || \
state == tsd_state_reincarnated || \
state == tsd_state_minimal_initialized); \
return tsd_##n##p_get_unsafe(tsd); \
@@ -94,6 +95,11 @@ TSD_DATA_SLOWER
JEMALLOC_ALWAYS_INLINE void
tsd_assert_fast(tsd_t *tsd) {
/*
* Note that our fastness assertion does *not* include global slowness
* counters; it's not in general possible to ensure that they won't
* change asynchronously from underneath us.
*/
assert(!malloc_slow && tsd_tcache_enabled_get(tsd)
&& tsd_reentrancy_level_get(tsd) == 0);
}

24
include/jemalloc/internal/tsd_binshards.h
View File

@@ -1,24 +0,0 @@
#ifndef JEMALLOC_INTERNAL_TSD_BINSHARDS_H
#define JEMALLOC_INTERNAL_TSD_BINSHARDS_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/sc.h"
/*
* Per-thread cache of bin-shard assignments. This lives in its own header
* (separate from bin.h) so that tsd_internals.h can pull it in for X-macro
* expansion without dragging in mutex.h, which itself depends on TSD machinery
* and would form an include-order dependency cycle.
*/
#define TSD_BINSHARDS_ZERO_INITIALIZER \
{ \
{ UINT8_MAX } \
}
typedef struct tsd_binshards_s tsd_binshards_t;
struct tsd_binshards_s {
uint8_t binshard[SC_NBINS];
};
#endif /* JEMALLOC_INTERNAL_TSD_BINSHARDS_H */

87
include/jemalloc/internal/tsd_internals.h
View File

@@ -4,27 +4,21 @@
#define JEMALLOC_INTERNAL_TSD_INTERNALS_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena_decay_constants.h"
#include "jemalloc/internal/arena_types.h"
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/tsd_binshards.h"
#include "jemalloc/internal/bin_types.h"
#include "jemalloc/internal/jemalloc_internal_externs.h"
#include "jemalloc/internal/peak.h"
#include "jemalloc/internal/prof_types.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/rtree_tsd.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/tcache_structs.h"
#include "jemalloc/internal/tcache_types.h"
#include "jemalloc/internal/thread_event_registry.h"
#include "jemalloc/internal/tsd_types.h"
#include "jemalloc/internal/util.h"
#include "jemalloc/internal/witness.h"
/*
* Forward decls. tsd_internals.h cannot include arena.h / prof.h directly:
* those headers' STRUCTS-section includes trigger mutex.h -> tsd.h ->
* tsd_generic.h, which would re-enter this file before its body finishes.
* Each consumer here only uses these as pointer types.
*/
typedef struct arena_s arena_t;
typedef struct prof_tdata_s prof_tdata_t;
/*
* Thread-Specific-Data layout
*
@@ -63,6 +57,8 @@ typedef void (*test_callback_t)(int *);
# define MALLOC_TEST_TSD_INITIALIZER
#endif
typedef ql_elm(tsd_t) tsd_link_t;
/* O(name, type, nullable type) */
#define TSD_DATA_SLOW \
O(tcache_enabled, bool, bool) \
@@ -84,6 +80,8 @@ typedef void (*test_callback_t)(int *);
O(arena_decay_ticker, ticker_geom_t, ticker_geom_t) \
O(sec_shard, uint8_t, uint8_t) \
O(binshards, tsd_binshards_t, tsd_binshards_t) \
O(tsd_link, tsd_link_t, tsd_link_t) \
O(in_hook, bool, bool) \
O(peak, peak_t, peak_t) \
O(tcache_slow, tcache_slow_t, tcache_slow_t) \
O(rtree_ctx, rtree_ctx_t, rtree_ctx_t)
@@ -103,8 +101,9 @@ typedef void (*test_callback_t)(int *);
TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE), \
/* sec_shard */ (uint8_t) - 1, \
/* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, \
/* peak */ PEAK_INITIALIZER, /* tcache_slow */ \
TCACHE_SLOW_ZERO_INITIALIZER, \
/* tsd_link */ {NULL}, /* in_hook */ false, \
/* peak */ PEAK_INITIALIZER, \
/* tcache_slow */ TCACHE_SLOW_ZERO_INITIALIZER, \
/* rtree_ctx */ RTREE_CTX_INITIALIZER,
/* O(name, type, nullable type) */
@@ -133,7 +132,7 @@ typedef void (*test_callback_t)(int *);
#define TSD_INITIALIZER \
{ \
TSD_DATA_SLOW_INITIALIZER \
/* state */ tsd_state_uninitialized, \
/* state */ ATOMIC_INIT(tsd_state_uninitialized), \
TSD_DATA_FAST_INITIALIZER TSD_DATA_SLOWER_INITIALIZER \
}
@@ -149,6 +148,17 @@ void tsd_cleanup(void *arg);
tsd_t *tsd_fetch_slow(tsd_t *tsd, bool minimal);
void tsd_state_set(tsd_t *tsd, uint8_t new_state);
void tsd_slow_update(tsd_t *tsd);
void tsd_prefork(tsd_t *tsd);
void tsd_postfork_parent(tsd_t *tsd);
void tsd_postfork_child(tsd_t *tsd);
/*
* Call ..._inc when your module wants to take all threads down the slow paths,
* and ..._dec when it no longer needs to.
*/
void tsd_global_slow_inc(tsdn_t *tsdn);
void tsd_global_slow_dec(tsdn_t *tsdn);
bool tsd_global_slow(void);
#define TSD_MIN_INIT_STATE_MAX_FETCHED (128)
@@ -157,29 +167,39 @@ enum {
tsd_state_nominal = 0,
/* Initialized but on slow path. */
tsd_state_nominal_slow = 1,
/*
* Some thread has changed global state in such a way that all nominal
* threads need to recompute their fast / slow status the next time they
* get a chance.
*
* Any thread can change another thread's status *to* recompute, but
* threads are the only ones who can change their status *from*
* recompute.
*/
tsd_state_nominal_recompute = 2,
/*
* The above nominal states should be lower values. We use
* tsd_nominal_max to separate nominal states from threads in the
* process of being born / dying.
*/
tsd_state_nominal_max = 1,
tsd_state_nominal_max = 2,
/*
* A thread might free() during its death as its only allocator action;
* in such scenarios, we need tsd, but set up in such a way that no
* cleanup is necessary.
*/
tsd_state_minimal_initialized = 2,
tsd_state_minimal_initialized = 3,
/* States during which we know we're in thread death. */
tsd_state_purgatory = 3,
tsd_state_reincarnated = 4,
tsd_state_purgatory = 4,
tsd_state_reincarnated = 5,
/*
* What it says on the tin; tsd that hasn't been initialized. Note
* that even when the tsd struct lives in TLS, when need to keep track
* of stuff like whether or not our pthread destructors have been
* scheduled, so this really truly is different than the nominal state.
*/
tsd_state_uninitialized = 5
tsd_state_uninitialized = 6
};
/*
@@ -189,6 +209,18 @@ enum {
*/
#define TSD_MANGLE(n) cant_access_tsd_items_directly_use_a_getter_or_setter_##n
#ifdef JEMALLOC_U8_ATOMICS
# define tsd_state_t atomic_u8_t
# define tsd_atomic_load atomic_load_u8
# define tsd_atomic_store atomic_store_u8
# define tsd_atomic_exchange atomic_exchange_u8
#else
# define tsd_state_t atomic_u32_t
# define tsd_atomic_load atomic_load_u32
# define tsd_atomic_store atomic_store_u32
# define tsd_atomic_exchange atomic_exchange_u32
#endif
/* The actual tsd. */
struct tsd_s {
/*
@@ -200,8 +232,11 @@ struct tsd_s {
#define O(n, t, nt) t TSD_MANGLE(n);
TSD_DATA_SLOW
/* Encodes one of tsd_state_*; mutated only by the owning thread. */
uint8_t state;
/*
* We manually limit the state to just a single byte. Unless the 8-bit
* atomics are unavailable (which is rare).
*/
tsd_state_t state;
TSD_DATA_FAST
TSD_DATA_SLOWER
#undef O
@@ -209,7 +244,13 @@ struct tsd_s {
JEMALLOC_ALWAYS_INLINE uint8_t
tsd_state_get(tsd_t *tsd) {
return tsd->state;
/*
* This should be atomic. Unfortunately, compilers right now can't tell
* that this can be done as a memory comparison, and forces a load into
* a register that hurts fast-path performance.
*/
/* return atomic_load_u8(&tsd->state, ATOMIC_RELAXED); */
return *(uint8_t *)&tsd->state;
}
/*

12
include/jemalloc/internal/util.h
View File

@@ -6,18 +6,6 @@
#define UTIL_INLINE static inline
JEMALLOC_ALWAYS_INLINE
size_t
min_zu(size_t a, size_t b) {
return (a < b) ? a : b;
}
JEMALLOC_ALWAYS_INLINE
size_t
max_zu(size_t a, size_t b) {
return (a > b) ? a : b;
}
/* Junk fill patterns. */
#ifndef JEMALLOC_ALLOC_JUNK
# define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5)

2
include/jemalloc/internal/witness.h
View File

@@ -44,7 +44,7 @@ enum witness_rank_e {
*/
WITNESS_RANK_CORE,
WITNESS_RANK_DECAY = WITNESS_RANK_CORE,
WITNESS_RANK_CACHE_BIN_ARRAY_DESCRIPTOR_QL,
WITNESS_RANK_TCACHE_QL,
WITNESS_RANK_SEC_BIN,

30
include/jemalloc/jemalloc_typedefs.h.in
View File

@@ -4,36 +4,6 @@ extern "C" {
typedef struct extent_hooks_s extent_hooks_t;
/*
* Extent alloc flags. A custom extent_alloc hook may OR these into the
* returned pointer; jemalloc strips the low bits before use. Safe because
* returned addresses are at least page-aligned (PAGE >= 256).
*
* EXTENT_ALLOC_FLAG_PINNED: backing memory is non-reclaimable.
* Pinned extents are excluded from decay/purging and cached separately for
* preferential reuse. A hook returning this flag must also set *commit to
* true: pinned memory bypasses jemalloc's commit/decommit machinery.
*
* The pinned attribute is per-extent: a single hook may return pinned and
* non-pinned extents in different calls. Pinned and non-pinned extents are
* never merged together (the merge would change the reclamation policy of
* one half), so pinned-ness is set at allocation and inherited through
* splits, but never changes after that.
*
* Example (HugeTLB alloc hook):
* void *my_alloc(extent_hooks_t *h, void *new_addr, size_t size,
* size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
* void *addr = mmap(NULL, size, PROT_READ|PROT_WRITE,
* MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0);
* if (addr == MAP_FAILED) return NULL;
* *zero = true;
* *commit = true;
* return (void *)((uintptr_t)addr | EXTENT_ALLOC_FLAG_PINNED);
* }
*/
#define EXTENT_ALLOC_FLAG_PINNED 0x1U
#define EXTENT_ALLOC_FLAG_MASK 0xFFU
/*
* void *
* extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,

5
msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
View File

@@ -36,7 +36,6 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\..\src\arena.c" />
<ClCompile Include="..\..\..\..\src\arenas_management.c" />
<ClCompile Include="..\..\..\..\src\background_thread.c" />
<ClCompile Include="..\..\..\..\src\base.c" />
<ClCompile Include="..\..\..\..\src\bin.c" />
@@ -45,7 +44,6 @@
<ClCompile Include="..\..\..\..\src\buf_writer.c" />
<ClCompile Include="..\..\..\..\src\cache_bin.c" />
<ClCompile Include="..\..\..\..\src\ckh.c" />
<ClCompile Include="..\..\..\..\src\conf.c" />
<ClCompile Include="..\..\..\..\src\counter.c" />
<ClCompile Include="..\..\..\..\src\ctl.c" />
<ClCompile Include="..\..\..\..\src\decay.c" />
@@ -69,11 +67,8 @@
<ClCompile Include="..\..\..\..\src\hpdata.c" />
<ClCompile Include="..\..\..\..\src\inspect.c" />
<ClCompile Include="..\..\..\..\src\jemalloc.c" />
<ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
<ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
<ClCompile Include="..\..\..\..\src\large.c" />
<ClCompile Include="..\..\..\..\src\log.c" />
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
<ClCompile Include="..\..\..\..\src\malloc_io.c" />
<ClCompile Include="..\..\..\..\src\mutex.c" />
<ClCompile Include="..\..\..\..\src\nstime.c" />

15
msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
View File

@@ -31,9 +31,6 @@
<ClCompile Include="..\..\..\..\src\ckh.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\conf.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\counter.c">
<Filter>Source Files</Filter>
</ClCompile>
@@ -94,9 +91,6 @@
<ClCompile Include="..\..\..\..\src\log.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_io.c">
<Filter>Source Files</Filter>
</ClCompile>
@@ -208,14 +202,5 @@
<ClCompile Include="..\..\..\..\src\san_bump.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\arenas_management.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\jemalloc_init.c">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
</Project>

5
msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
View File

@@ -36,7 +36,6 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\..\src\arena.c" />
<ClCompile Include="..\..\..\..\src\arenas_management.c" />
<ClCompile Include="..\..\..\..\src\background_thread.c" />
<ClCompile Include="..\..\..\..\src\base.c" />
<ClCompile Include="..\..\..\..\src\bin.c" />
@@ -45,7 +44,6 @@
<ClCompile Include="..\..\..\..\src\buf_writer.c" />
<ClCompile Include="..\..\..\..\src\cache_bin.c" />
<ClCompile Include="..\..\..\..\src\ckh.c" />
<ClCompile Include="..\..\..\..\src\conf.c" />
<ClCompile Include="..\..\..\..\src\counter.c" />
<ClCompile Include="..\..\..\..\src\ctl.c" />
<ClCompile Include="..\..\..\..\src\decay.c" />
@@ -69,11 +67,8 @@
<ClCompile Include="..\..\..\..\src\hpdata.c" />
<ClCompile Include="..\..\..\..\src\inspect.c" />
<ClCompile Include="..\..\..\..\src\jemalloc.c" />
<ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
<ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
<ClCompile Include="..\..\..\..\src\large.c" />
<ClCompile Include="..\..\..\..\src\log.c" />
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
<ClCompile Include="..\..\..\..\src\malloc_io.c" />
<ClCompile Include="..\..\..\..\src\mutex.c" />
<ClCompile Include="..\..\..\..\src\nstime.c" />

15
msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
View File

@@ -31,9 +31,6 @@
<ClCompile Include="..\..\..\..\src\ckh.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\conf.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\counter.c">
<Filter>Source Files</Filter>
</ClCompile>
@@ -94,9 +91,6 @@
<ClCompile Include="..\..\..\..\src\log.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_io.c">
<Filter>Source Files</Filter>
</ClCompile>
@@ -208,14 +202,5 @@
<ClCompile Include="..\..\..\..\src\san_bump.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\arenas_management.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\jemalloc_init.c">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
</Project>

5
msvc/projects/vc2019/jemalloc/jemalloc.vcxproj
View File

@@ -36,7 +36,6 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\..\src\arena.c" />
<ClCompile Include="..\..\..\..\src\arenas_management.c" />
<ClCompile Include="..\..\..\..\src\background_thread.c" />
<ClCompile Include="..\..\..\..\src\base.c" />
<ClCompile Include="..\..\..\..\src\bin.c" />
@@ -45,7 +44,6 @@
<ClCompile Include="..\..\..\..\src\buf_writer.c" />
<ClCompile Include="..\..\..\..\src\cache_bin.c" />
<ClCompile Include="..\..\..\..\src\ckh.c" />
<ClCompile Include="..\..\..\..\src\conf.c" />
<ClCompile Include="..\..\..\..\src\counter.c" />
<ClCompile Include="..\..\..\..\src\ctl.c" />
<ClCompile Include="..\..\..\..\src\decay.c" />
@@ -69,11 +67,8 @@
<ClCompile Include="..\..\..\..\src\hpdata.c" />
<ClCompile Include="..\..\..\..\src\inspect.c" />
<ClCompile Include="..\..\..\..\src\jemalloc.c" />
<ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
<ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
<ClCompile Include="..\..\..\..\src\large.c" />
<ClCompile Include="..\..\..\..\src\log.c" />
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
<ClCompile Include="..\..\..\..\src\malloc_io.c" />
<ClCompile Include="..\..\..\..\src\mutex.c" />
<ClCompile Include="..\..\..\..\src\nstime.c" />

15
msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters
View File

@@ -31,9 +31,6 @@
<ClCompile Include="..\..\..\..\src\ckh.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\conf.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\counter.c">
<Filter>Source Files</Filter>
</ClCompile>
@@ -94,9 +91,6 @@
<ClCompile Include="..\..\..\..\src\log.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_io.c">
<Filter>Source Files</Filter>
</ClCompile>
@@ -208,14 +202,5 @@
<ClCompile Include="..\..\..\..\src\san_bump.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\arenas_management.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\jemalloc_init.c">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
</Project>

5
msvc/projects/vc2022/jemalloc/jemalloc.vcxproj
View File

@@ -36,7 +36,6 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\..\src\arena.c" />
<ClCompile Include="..\..\..\..\src\arenas_management.c" />
<ClCompile Include="..\..\..\..\src\background_thread.c" />
<ClCompile Include="..\..\..\..\src\base.c" />
<ClCompile Include="..\..\..\..\src\bin.c" />
@@ -45,7 +44,6 @@
<ClCompile Include="..\..\..\..\src\buf_writer.c" />
<ClCompile Include="..\..\..\..\src\cache_bin.c" />
<ClCompile Include="..\..\..\..\src\ckh.c" />
<ClCompile Include="..\..\..\..\src\conf.c" />
<ClCompile Include="..\..\..\..\src\counter.c" />
<ClCompile Include="..\..\..\..\src\ctl.c" />
<ClCompile Include="..\..\..\..\src\decay.c" />
@@ -69,11 +67,8 @@
<ClCompile Include="..\..\..\..\src\hpdata.c" />
<ClCompile Include="..\..\..\..\src\inspect.c" />
<ClCompile Include="..\..\..\..\src\jemalloc.c" />
<ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
<ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
<ClCompile Include="..\..\..\..\src\large.c" />
<ClCompile Include="..\..\..\..\src\log.c" />
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
<ClCompile Include="..\..\..\..\src\malloc_io.c" />
<ClCompile Include="..\..\..\..\src\mutex.c" />
<ClCompile Include="..\..\..\..\src\nstime.c" />

15
msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters
View File

@@ -31,9 +31,6 @@
<ClCompile Include="..\..\..\..\src\ckh.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\conf.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\counter.c">
<Filter>Source Files</Filter>
</ClCompile>
@@ -94,9 +91,6 @@
<ClCompile Include="..\..\..\..\src\log.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_io.c">
<Filter>Source Files</Filter>
</ClCompile>
@@ -208,14 +202,5 @@
<ClCompile Include="..\..\..\..\src\san_bump.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\arenas_management.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\jemalloc_init.c">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
</Project>

72
scripts/gen_gh_actions.py
View File

@@ -276,24 +276,6 @@ def generate_linux_job(arch):
}
]
# --enable-cxx-infallible-new coverage. Plain variant runs on all arches;
# debug-combined variant runs only on AMD64 to bound matrix size.
infallible_new_entries = [
{
'CC': 'gcc',
'CXX': 'g++',
'CONFIGURE_FLAGS': '--enable-cxx-infallible-new',
'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
},
]
if arch == AMD64:
infallible_new_entries.append({
'CC': 'gcc',
'CXX': 'g++',
'CONFIGURE_FLAGS': '--enable-cxx-infallible-new --enable-debug',
'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
})
if arch == AMD64:
for entry in manual_entries:
job += " - env:\n"
@@ -303,17 +285,9 @@ def generate_linux_job(arch):
else:
job += f" {key}: {value}\n"
for entry in infallible_new_entries:
job += " - env:\n"
for key, value in entry.items():
if ' ' in str(value):
job += f' {key}: "{value}"\n'
else:
job += f" {key}: {value}\n"
job += f"""
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Show OS version
run: |
@@ -328,7 +302,7 @@ def generate_linux_job(arch):
cat /etc/os-release || true
echo ""
echo "=== CPU Info ==="
lscpu | grep -E "Architecture|CPU op-mode|Byte Order|CPU\\(s\\):" || true
lscpu | grep -E "Architecture|CPU op-mode|Byte Order|CPU\(s\):" || true
- name: Install dependencies (32-bit)
if: matrix.env.CROSS_COMPILE_32BIT == 'yes'
@@ -411,27 +385,9 @@ def generate_macos_job(arch):
else:
job += f" {key}: {value}\n"
# --enable-cxx-infallible-new coverage on macOS (both arches).
macos_extra_cflags = ' '.join(get_extra_cflags(OSX, GCC.value))
infallible_new_entries = [
{
'CC': 'gcc',
'CXX': 'g++',
'CONFIGURE_FLAGS': '--enable-cxx-infallible-new',
'EXTRA_CFLAGS': macos_extra_cflags
},
]
for entry in infallible_new_entries:
job += " - env:\n"
for key, value in entry.items():
if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
job += f' {key}: "{value}"\n'
else:
job += f" {key}: {value}\n"
job += f"""
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Show OS version
run: |
@@ -511,27 +467,9 @@ def generate_windows_job(arch):
else:
job += f" {key}: {value}\n"
# --enable-cxx-infallible-new coverage on Windows (MinGW-GCC only).
windows_mingw_cflags = ' '.join(get_extra_cflags(WINDOWS, GCC.value))
infallible_new_entries = [
{
'CC': 'gcc',
'CXX': 'g++',
'CONFIGURE_FLAGS': '--enable-cxx-infallible-new',
'EXTRA_CFLAGS': windows_mingw_cflags
},
]
for entry in infallible_new_entries:
job += " - env:\n"
for key, value in entry.items():
if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
job += f' {key}: "{value}"\n'
else:
job += f" {key}: {value}\n"
job += f"""
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- name: Show OS version
shell: cmd
@@ -652,7 +590,7 @@ def generate_freebsd_job(arch):
name: FreeBSD (${{{{ matrix.arch }}}}, debug=${{{{ matrix.debug }}}}, prof=${{{{ matrix.prof }}}}${{{{ matrix.uncommon && ', uncommon' || '' }}}})
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
with:
fetch-depth: 1

100
scripts/gen_header_dep_graph.sh
View File

@@ -1,100 +0,0 @@
#!/bin/bash
# Generate header dependency graph + tsort cycle check.
#
# Outputs (under build/):
# header_deps_baseline.txt - sorted, unique header->header (and .c->header)
# edges derived from #include "..." directives.
# header_tsort_order.txt - topological order produced by tsort.
# header_tsort_cycles.txt - stderr from tsort; non-empty if there's a cycle.
# per_tu_deps.txt - per-translation-unit transitive header lists
# (one TU per line; basenames only; sorted).
#
# All headers / sources are reduced to basenames so renames in the cleanup are
# easy to spot in the diff.
set -euo pipefail
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
cd "$repo_root"
mkdir -p build
edges_raw=build/header_edges_raw.txt
edges_baseline=build/header_deps_baseline.txt
tsort_order=build/header_tsort_order.txt
tsort_cycles=build/header_tsort_cycles.txt
per_tu=build/per_tu_deps.txt
: > "$edges_raw"
extract_edges() {
local src="$1"
local bn
bn=$(basename "$src")
# Match `#include "..."` (skip `<...>` system includes). Strip path, keep
# only the basename of the included file, so the graph collapses
# `../jemalloc.h` and similar relative paths into a single node.
awk '
/^[[:space:]]*#[[:space:]]*include[[:space:]]+"/ {
match($0, /"[^"]+"/)
inc = substr($0, RSTART + 1, RLENGTH - 2)
n = split(inc, parts, "/")
print parts[n]
}
' "$src" | while read -r dep; do
printf '%s %s\n' "$bn" "$dep"
done
}
# Headers (include/jemalloc/internal/) — these are the nodes we care about for
# cycle detection.
for f in include/jemalloc/internal/*.h include/jemalloc/*.h; do
extract_edges "$f" >> "$edges_raw"
done
# Translation units (src/*.c) — included so that .c files appear as graph
# sources in the baseline. They can't introduce cycles (nothing #includes a
# .c), but the edges are useful when diffing later.
for f in src/*.c; do
extract_edges "$f" >> "$edges_raw"
done
sort -u "$edges_raw" > "$edges_baseline"
# tsort consumes "A B" pairs and reports cycles on stderr.
: > "$tsort_cycles"
if ! tsort "$edges_baseline" > "$tsort_order" 2> "$tsort_cycles"; then
echo "tsort exited non-zero; see $tsort_cycles" >&2
fi
# Per-translation-unit transitive header lists, harvested from the .d files
# the build already produced (CC_MM=1 in the Makefile).
: > "$per_tu"
for d in src/*.d; do
[ -f "$d" ] || continue
tu=$(basename "${d%.d}.c")
# Strip the make rule prefix ("foo.o: foo.c \"), drop line continuations,
# collapse to basenames, sort+uniq.
deps=$(
tr '\n' ' ' < "$d" \
| sed -E 's/\\//g' \
| tr ' ' '\n' \
| grep -E '\.h$' \
| awk -F/ '{print $NF}' \
| sort -u \
| tr '\n' ' ' \
| sed -E 's/[[:space:]]+$//'
)
printf '%s: %s\n' "$tu" "$deps" >> "$per_tu"
done
sort -o "$per_tu" "$per_tu"
edge_count=$(wc -l < "$edges_baseline" | tr -d ' ')
cycle_bytes=$(wc -c < "$tsort_cycles" | tr -d ' ')
tu_count=$(wc -l < "$per_tu" | tr -d ' ')
echo "edges: $edge_count"
echo "translation units: $tu_count"
echo "tsort cycle output bytes: $cycle_bytes"
if [ "$cycle_bytes" -gt 0 ]; then
echo "---- tsort cycle report ----"
cat "$tsort_cycles"
fi

396
src/arena.c
View File

@@ -1,27 +1,16 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/arena_inlines.h"
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/background_thread.h"
#include "jemalloc/internal/background_thread_inlines.h"
#include "jemalloc/internal/bin_inlines.h"
#include "jemalloc/internal/decay.h"
#include "jemalloc/internal/ehooks.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/extent_mmap.h"
#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
#include "jemalloc/internal/jemalloc_internal_inlines_c.h"
#include "jemalloc/internal/large.h"
#include "jemalloc/internal/san.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/prof.h"
#include "jemalloc/internal/prof_inlines.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/safety_check.h"
#include "jemalloc/internal/san.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/util.h"
#include "jemalloc/internal/witness.h"
JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
@@ -29,7 +18,7 @@ JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
/* Data. */
/*
* Define names for both uninitialized and initialized phases, so that
* Define names for both unininitialized and initialized phases, so that
* options and mallctl processing are straightforward.
*/
const char *const percpu_arena_mode_names[] = {
@@ -47,14 +36,6 @@ static pa_central_t arena_pa_central_global;
div_info_t arena_binind_div_info[SC_NBINS];
JET_EXTERN void
bin_dalloc_locked_begin(
bin_dalloc_locked_info_t *info, szind_t binind) {
info->div_info = arena_binind_div_info[binind];
info->nregs = bin_infos[binind].nregs;
info->ndalloc = 0;
}
size_t opt_oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT;
size_t oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT;
@@ -63,10 +44,10 @@ static unsigned nbins_total;
/*
* a0 is used to handle huge requests before malloc init completes. After
* that, the huge_arena_ind is updated to point to the actual huge arena,
* that,the huge_arena_ind is updated to point to the actual huge arena,
* which is the last one of the auto arenas.
*/
static unsigned huge_arena_ind = 0;
unsigned huge_arena_ind = 0;
bool opt_huge_arena_pac_thp = false;
pac_thp_t huge_arena_pac_thp = {.thp_madvise = false,
.auto_thp_switched = false,
@@ -183,7 +164,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
/* Currently cached bytes and sanitizer-stashed bytes in tcache. */
astats->tcache_bytes = 0;
astats->tcache_stashed_bytes = 0;
malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
cache_bin_array_descriptor_t *descriptor;
ql_foreach (descriptor, &arena->cache_bin_array_descriptor_ql, link) {
for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) {
@@ -202,8 +183,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
}
malloc_mutex_prof_read(tsdn,
&astats->mutex_prof_data[arena_prof_mutex_tcache_list],
&arena->cache_bin_array_descriptor_ql_mtx);
malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
&arena->tcache_ql_mtx);
malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
#define READ_ARENA_MUTEX_PROF_DATA(mtx, ind) \
malloc_mutex_lock(tsdn, &arena->mtx); \
@@ -230,92 +211,6 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
}
}
void *
arena_locality_hint(tsdn_t *tsdn, arena_t *arena, szind_t szind) {
assert(szind < SC_NBINS);
bin_t *bin = bin_choose(tsdn, arena, szind, NULL);
assert(bin != NULL);
return bin_current_slab_addr(tsdn, bin);
}
void
arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena,
cache_bin_array_descriptor_t *desc) {
cassert(config_stats);
malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
ql_tail_insert(&arena->cache_bin_array_descriptor_ql, desc, link);
malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
}
static void
arena_cache_bin_stats_flush(tsdn_t *tsdn, arena_t *arena,
cache_bin_array_descriptor_t *desc) {
cassert(config_stats);
for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) {
cache_bin_t *cache_bin = &desc->bins[i];
if (cache_bin_disabled(cache_bin)) {
continue;
}
if (i < SC_NBINS) {
bin_t *bin = bin_choose(tsdn, arena, i, NULL);
bin_stats_nrequests_add(tsdn, bin,
cache_bin->tstats.nrequests);
} else {
arena_stats_large_flush_nrequests_add(tsdn,
&arena->stats, i, cache_bin->tstats.nrequests);
}
cache_bin->tstats.nrequests = 0;
}
}
void
arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
cache_bin_array_descriptor_t *desc) {
cassert(config_stats);
malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
if (config_debug) {
bool in_ql = false;
cache_bin_array_descriptor_t *iter;
ql_foreach (iter, &arena->cache_bin_array_descriptor_ql, link) {
if (iter == desc) {
in_ql = true;
break;
}
}
assert(in_ql);
}
ql_remove(&arena->cache_bin_array_descriptor_ql, desc, link);
arena_cache_bin_stats_flush(tsdn, arena, desc);
malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
}
/*
* Postfork-child entry: child is single-threaded, so the queue is rebuilt
* from scratch (descriptors held by other threads at fork time are gone)
* without locking.
*/
void
arena_cache_bin_array_postfork_child(arena_t *arena,
cache_bin_array_descriptor_t *desc_or_null) {
cassert(config_stats);
ql_new(&arena->cache_bin_array_descriptor_ql);
if (desc_or_null != NULL) {
ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
desc_or_null, link);
}
}
void
arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena) {
cassert(config_stats);
malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
cache_bin_array_descriptor_t *desc;
ql_foreach (desc, &arena->cache_bin_array_descriptor_ql, link) {
arena_cache_bin_stats_flush(tsdn, arena, desc);
}
malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
}
static void
arena_background_thread_inactivity_check(
tsdn_t *tsdn, arena_t *arena, bool is_background_thread) {
@@ -448,8 +343,7 @@ arena_extent_alloc_large(
}
void
arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena,
const edata_t *edata) {
arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
if (config_stats) {
arena_large_dalloc_stats_update(
tsdn, arena, edata_usize_get(edata));
@@ -458,7 +352,7 @@ arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena,
void
arena_extent_ralloc_large_shrink(
tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize) {
tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize) {
size_t usize = edata_usize_get(edata);
if (config_stats) {
@@ -468,7 +362,7 @@ arena_extent_ralloc_large_shrink(
void
arena_extent_ralloc_large_expand(
tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize) {
tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize) {
size_t usize = edata_usize_get(edata);
if (config_stats) {
@@ -713,13 +607,11 @@ arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize) {
assert(isalloc(tsdn, ptr) == usize);
}
size_t
static size_t
arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
cassert(config_prof);
assert(opt_prof);
assert(ptr != NULL);
size_t usize = edata_usize_get(edata);
assert(isalloc(tsdn, ptr) == usize);
size_t usize = isalloc(tsdn, ptr);
size_t bumped_usize = sz_sa2u(usize, PROF_SAMPLE_ALIGNMENT);
assert(bumped_usize <= SC_LARGE_MINCLASS
&& PAGE_CEILING(bumped_usize) == bumped_usize);
@@ -731,6 +623,17 @@ arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
assert(isalloc(tsdn, ptr) == bumped_usize);
return bumped_usize;
}
static void
arena_dalloc_promoted_impl(
tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path, edata_t *edata) {
cassert(config_prof);
assert(opt_prof);
size_t usize = edata_usize_get(edata);
size_t bumped_usize = arena_prof_demote(tsdn, edata, ptr);
if (config_opt_safety_checks && usize < SC_LARGE_MINCLASS) {
/*
* Currently, we only do redzoning for small sampled
@@ -738,8 +641,23 @@ arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
*/
safety_check_verify_redzone(ptr, usize, bumped_usize);
}
szind_t bumped_ind = sz_size2index(bumped_usize);
if (bumped_usize >= SC_LARGE_MINCLASS && tcache != NULL
&& bumped_ind < TCACHE_NBINS_MAX
&& !tcache_bin_disabled(
bumped_ind, &tcache->bins[bumped_ind], tcache->tcache_slow)) {
tcache_dalloc_large(
tsdn_tsd(tsdn), tcache, ptr, bumped_ind, slow_path);
} else {
large_dalloc(tsdn, edata);
}
}
return bumped_usize;
void
arena_dalloc_promoted(
tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
arena_dalloc_promoted_impl(tsdn, ptr, tcache, slow_path, edata);
}
void
@@ -781,8 +699,8 @@ arena_reset(tsd_t *tsd, arena_t *arena) {
prof_free(tsd, ptr, usize, &alloc_ctx);
}
if (config_prof && opt_prof && alloc_ctx.szind < SC_NBINS) {
arena_prof_demote(tsd_tsdn(tsd), edata, ptr);
large_dalloc(tsd_tsdn(tsd), edata);
arena_dalloc_promoted_impl(tsd_tsdn(tsd), ptr,
/* tcache */ NULL, /* slow_path */ true, edata);
} else {
large_dalloc(tsd_tsdn(tsd), edata);
}
@@ -878,8 +796,6 @@ arena_prepare_base_deletion(tsd_t *tsd, base_t *base_to_destroy) {
tsd, &pac->ecache_muzzy.mtx, delayed_mtx, &n_delayed);
arena_prepare_base_deletion_sync(
tsd, &pac->ecache_retained.mtx, delayed_mtx, &n_delayed);
arena_prepare_base_deletion_sync(
tsd, &pac->ecache_pinned.mtx, delayed_mtx, &n_delayed);
}
arena_prepare_base_deletion_sync_finish(tsd, delayed_mtx, n_delayed);
}
@@ -1081,6 +997,76 @@ label_refill:
return filled;
}
size_t
arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
void **ptrs, size_t nfill, bool zero) {
assert(binind < SC_NBINS);
const bin_info_t *bin_info = &bin_infos[binind];
const size_t nregs = bin_info->nregs;
assert(nregs > 0);
const size_t usize = bin_info->reg_size;
const bool manual_arena = !arena_is_auto(arena);
unsigned binshard;
bin_t *bin = bin_choose(tsdn, arena, binind, &binshard);
size_t nslab = 0;
size_t filled = 0;
edata_t *slab = NULL;
edata_list_active_t fulls;
edata_list_active_init(&fulls);
while (filled < nfill
&& (slab = arena_slab_alloc(
tsdn, arena, binind, binshard, bin_info))
!= NULL) {
assert((size_t)edata_nfree_get(slab) == nregs);
++nslab;
size_t batch = nfill - filled;
if (batch > nregs) {
batch = nregs;
}
assert(batch > 0);
bin_slab_reg_alloc_batch(
slab, bin_info, (unsigned)batch, &ptrs[filled]);
assert(edata_addr_get(slab) == ptrs[filled]);
if (zero) {
memset(ptrs[filled], 0, batch * usize);
}
filled += batch;
if (batch == nregs) {
if (manual_arena) {
edata_list_active_append(&fulls, slab);
}
slab = NULL;
}
}
malloc_mutex_lock(tsdn, &bin->lock);
/*
* Only the last slab can be non-empty, and the last slab is non-empty
* iff slab != NULL.
*/
if (slab != NULL) {
bin_lower_slab(tsdn, !manual_arena, slab, bin);
}
if (manual_arena) {
edata_list_active_concat(&bin->slabs_full, &fulls);
}
assert(edata_list_active_empty(&fulls));
if (config_stats) {
bin->stats.nslabs += nslab;
bin->stats.curslabs += nslab;
bin->stats.nmalloc += filled;
bin->stats.nrequests += filled;
bin->stats.curregs += filled;
}
malloc_mutex_unlock(tsdn, &bin->lock);
arena_decay_tick(tsdn, arena);
return filled;
}
static void *
arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) {
assert(binind < SC_NBINS);
@@ -1151,6 +1137,33 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
}
}
void *
arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
bool zero, bool slab, tcache_t *tcache) {
if (slab) {
assert(sz_can_use_slab(usize));
/* Small; alignment doesn't require special slab placement. */
/* usize should be a result of sz_sa2u() */
assert((usize & (alignment - 1)) == 0);
/*
* Small usize can't come from an alignment larger than a page.
*/
assert(alignment <= PAGE);
return arena_malloc(tsdn, arena, usize, sz_size2index(usize),
zero, slab, tcache, true);
} else {
if (likely(alignment <= CACHELINE)) {
return large_malloc(tsdn, arena, usize, zero);
} else {
return large_palloc(
tsdn, arena, usize, alignment, zero);
}
}
}
static void
arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, edata_t *edata, void *ptr) {
szind_t binind = edata_szind_get(edata);
@@ -1484,10 +1497,10 @@ arena_ptr_array_flush_impl(tsd_t *tsd, szind_t binind,
* '...' is morally equivalent, the code itself needs slight tweaks.
*/
if (small) {
arena_ptr_array_flush_impl_small(tsdn, binind, arr,
return arena_ptr_array_flush_impl_small(tsdn, binind, arr,
item_edata, nflush, stats_arena, merge_stats);
} else {
arena_ptr_array_flush_impl_large(tsdn, binind, arr,
return arena_ptr_array_flush_impl_large(tsdn, binind, arr,
item_edata, nflush, stats_arena, merge_stats);
}
}
@@ -1577,8 +1590,79 @@ done:
return ret;
}
static void *
arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero, bool slab, tcache_t *tcache) {
if (alignment == 0) {
return arena_malloc(tsdn, arena, usize, sz_size2index(usize),
zero, slab, tcache, true);
}
usize = sz_sa2u(usize, alignment);
if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
return NULL;
}
return ipalloct_explicit_slab(
tsdn, usize, alignment, zero, slab, tcache, arena);
}
void *
arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache,
hook_ralloc_args_t *hook_args) {
size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment);
if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
return NULL;
}
if (likely(slab)) {
assert(sz_can_use_slab(usize));
/* Try to avoid moving the allocation. */
UNUSED size_t newsize;
if (!arena_ralloc_no_move(
tsdn, ptr, oldsize, usize, 0, zero, &newsize)) {
hook_invoke_expand(hook_args->is_realloc
? hook_expand_realloc
: hook_expand_rallocx,
ptr, oldsize, usize, (uintptr_t)ptr,
hook_args->args);
return ptr;
}
}
if (oldsize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS) {
return large_ralloc(tsdn, arena, ptr, usize, alignment, zero,
tcache, hook_args);
}
/*
* size and oldsize are different enough that we need to move the
* object. In that case, fall back to allocating new space and copying.
*/
void *ret = arena_ralloc_move_helper(
tsdn, arena, usize, alignment, zero, slab, tcache);
if (ret == NULL) {
return NULL;
}
hook_invoke_alloc(
hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx,
ret, (uintptr_t)ret, hook_args->args);
hook_invoke_dalloc(
hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx,
ptr, hook_args->args);
/*
* Junk/zero-filling were already done by
* ipalloc()/arena_malloc().
*/
size_t copysize = (usize < oldsize) ? usize : oldsize;
memcpy(ret, ptr, copysize);
isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
return ret;
}
ehooks_t *
arena_get_ehooks(const arena_t *arena) {
arena_get_ehooks(arena_t *arena) {
return base_ehooks_get(arena->base);
}
@@ -1601,7 +1685,7 @@ arena_set_extent_hooks(
}
dss_prec_t
arena_dss_prec_get(const arena_t *arena) {
arena_dss_prec_get(arena_t *arena) {
return (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_ACQUIRE);
}
@@ -1615,9 +1699,8 @@ arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) {
}
void
arena_name_get(const arena_t *arena, char *name) {
const char *end = (const char *)memchr(
arena->name, '\0', ARENA_NAME_LEN);
arena_name_get(arena_t *arena, char *name) {
char *end = (char *)memchr((void *)arena->name, '\0', ARENA_NAME_LEN);
assert(end != NULL);
size_t len = (uintptr_t)end - (uintptr_t)arena->name + 1;
assert(len > 0 && len <= ARENA_NAME_LEN);
@@ -1668,7 +1751,7 @@ arena_retain_grow_limit_get_set(
}
unsigned
arena_nthreads_get(const arena_t *arena, bool internal) {
arena_nthreads_get(arena_t *arena, bool internal) {
return atomic_load_u(&arena->nthreads[internal], ATOMIC_RELAXED);
}
@@ -1716,11 +1799,10 @@ arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
goto label_error;
}
ql_new(&arena->tcache_ql);
ql_new(&arena->cache_bin_array_descriptor_ql);
if (malloc_mutex_init(&arena->cache_bin_array_descriptor_ql_mtx,
"cache_bin_array_descriptor_ql",
WITNESS_RANK_CACHE_BIN_ARRAY_DESCRIPTOR_QL,
malloc_mutex_rank_exclusive)) {
if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql",
WITNESS_RANK_TCACHE_QL, malloc_mutex_rank_exclusive)) {
goto label_error;
}
}
@@ -1872,6 +1954,9 @@ arena_init_huge(tsdn_t *tsdn, arena_t *a0) {
huge_arena_ind = narenas_total_get();
assert(huge_arena_ind != 0);
oversize_threshold = opt_oversize_threshold;
/* a0 init happened before malloc_conf_init. */
atomic_store_zu(&a0->pa_shard.pac.oversize_threshold,
oversize_threshold, ATOMIC_RELAXED);
/* Initialize huge_arena_pac_thp fields. */
base_t *b0 = a0->base;
/* Make sure that b0 thp auto-switch won't happen concurrently here. */
@@ -1889,18 +1974,9 @@ arena_init_huge(tsdn_t *tsdn, arena_t *a0) {
huge_enabled = true;
}
/* a0 init happened before malloc_conf_init. */
atomic_store_zu(&a0->pa_shard.pac.oversize_threshold,
oversize_threshold, ATOMIC_RELAXED);
return huge_enabled;
}
bool
arena_ind_is_huge(unsigned ind) {
return huge_arena_ind != 0 && ind == huge_arena_ind;
}
bool
arena_boot(sc_data_t *sc_data, base_t *base, bool hpa) {
arena_dirty_decay_ms_default_set(opt_dirty_decay_ms);
@@ -1930,7 +2006,7 @@ arena_prefork0(tsdn_t *tsdn, arena_t *arena) {
void
arena_prefork1(tsdn_t *tsdn, arena_t *arena) {
if (config_stats) {
malloc_mutex_prefork(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
malloc_mutex_prefork(tsdn, &arena->tcache_ql_mtx);
}
}
@@ -1983,13 +2059,12 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
base_postfork_parent(tsdn, arena->base);
pa_shard_postfork_parent(tsdn, &arena->pa_shard);
if (config_stats) {
malloc_mutex_postfork_parent(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx);
}
}
void
arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
cache_bin_array_descriptor_t *surviving_desc) {
arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED);
atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED);
if (tsd_arena_get(tsdn_tsd(tsdn)) == arena) {
@@ -1999,9 +2074,19 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
arena_nthreads_inc(arena, true);
}
if (config_stats) {
malloc_mutex_postfork_child(tsdn,
&arena->cache_bin_array_descriptor_ql_mtx);
arena_cache_bin_array_postfork_child(arena, surviving_desc);
ql_new(&arena->tcache_ql);
ql_new(&arena->cache_bin_array_descriptor_ql);
tcache_slow_t *tcache_slow = tcache_slow_get(tsdn_tsd(tsdn));
if (tcache_slow != NULL && tcache_slow->arena == arena) {
tcache_t *tcache = tcache_slow->tcache;
ql_elm_new(tcache_slow, link);
ql_tail_insert(&arena->tcache_ql, tcache_slow, link);
cache_bin_array_descriptor_init(
&tcache_slow->cache_bin_array_descriptor,
tcache->bins);
ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
&tcache_slow->cache_bin_array_descriptor, link);
}
}
for (unsigned i = 0; i < nbins_total; i++) {
@@ -2012,4 +2097,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
base_postfork_child(tsdn, arena->base);
pa_shard_postfork_child(tsdn, &arena->pa_shard);
if (config_stats) {
malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx);
}
}

360
src/arenas_management.c
View File

@@ -1,360 +0,0 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/arenas_management.h"
#include "jemalloc/internal/background_thread.h"
#include "jemalloc/internal/background_thread_inlines.h"
#include "jemalloc/internal/jemalloc_init.h"
#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
#include "jemalloc/internal/arena_inlines.h"
#include "jemalloc/internal/jemalloc_internal_inlines_c.h"
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/tcache.h"
JEMALLOC_ALIGNED(CACHELINE)
atomic_p_t arenas[MALLOCX_ARENA_LIMIT];
/* Below two are read-only after initialization. */
unsigned narenas_auto;
unsigned manual_arena_base;
static atomic_u_t narenas_total;
static malloc_mutex_t arenas_lock;
bool
arenas_management_boot(void) {
return malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS,
malloc_mutex_rank_exclusive);
}
void
arenas_management_prefork(tsdn_t *tsdn) {
malloc_mutex_prefork(tsdn, &arenas_lock);
}
void
arenas_management_postfork_parent(tsdn_t *tsdn) {
malloc_mutex_postfork_parent(tsdn, &arenas_lock);
}
void
arenas_management_postfork_child(tsdn_t *tsdn) {
malloc_mutex_postfork_child(tsdn, &arenas_lock);
}
void
narenas_total_set(unsigned narenas) {
atomic_store_u(&narenas_total, narenas, ATOMIC_RELEASE);
}
void
narenas_total_inc(void) {
atomic_fetch_add_u(&narenas_total, 1, ATOMIC_RELEASE);
}
unsigned
narenas_total_get(void) {
return atomic_load_u(&narenas_total, ATOMIC_ACQUIRE);
}
void
narenas_auto_set(unsigned n) {
narenas_auto = n;
}
void
manual_arena_base_set(unsigned base) {
manual_arena_base = base;
}
/*
* The a0*() functions are used instead of i{d,}alloc() in situations that
* cannot tolerate TLS variable access.
*/
void *
a0ialloc(size_t size, bool zero, bool is_internal) {
if (unlikely(malloc_init_a0())) {
return NULL;
}
return iallocztm(TSDN_NULL, size, sz_size2index(size), zero, NULL,
is_internal, arena_get(TSDN_NULL, 0, true), true);
}
void
a0idalloc(void *ptr, bool is_internal) {
idalloctm(TSDN_NULL, ptr, NULL, NULL, is_internal, true);
}
void *
a0malloc(size_t size) {
return a0ialloc(size, false, true);
}
void
a0dalloc(void *ptr) {
a0idalloc(ptr, true);
}
void
arena_set(unsigned ind, arena_t *arena) {
atomic_store_p(&arenas[ind], arena, ATOMIC_RELEASE);
}
/* Create a new arena and insert it into the arenas array at index ind. */
static arena_t *
arena_init_locked(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
arena_t *arena;
assert(ind <= narenas_total_get());
if (ind >= MALLOCX_ARENA_LIMIT) {
return NULL;
}
if (ind == narenas_total_get()) {
narenas_total_inc();
}
/*
* Another thread may have already initialized arenas[ind] if it's an
* auto arena.
*/
arena = arena_get(tsdn, ind, false);
if (arena != NULL) {
assert(arena_is_auto(arena));
return arena;
}
/* Actually initialize the arena. */
arena = arena_new(tsdn, ind, config);
return arena;
}
static void
arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) {
if (ind == 0) {
return;
}
if (have_background_thread) {
if (background_thread_create(tsdn_tsd(tsdn), ind)) {
malloc_printf(
"<jemalloc>: error in background thread "
"creation for arena %u. Abort.\n",
ind);
abort();
}
}
}
arena_t *
arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
arena_t *arena;
malloc_mutex_lock(tsdn, &arenas_lock);
arena = arena_init_locked(tsdn, ind, config);
malloc_mutex_unlock(tsdn, &arenas_lock);
arena_new_create_background_thread(tsdn, ind);
return arena;
}
static void
arena_bind(tsd_t *tsd, unsigned ind, bool internal) {
arena_t *arena = arena_get(tsd_tsdn(tsd), ind, false);
arena_nthreads_inc(arena, internal);
if (internal) {
tsd_iarena_set(tsd, arena);
} else {
tsd_arena_set(tsd, arena);
/*
* While shard acts as a random seed, the cast below should
* not make much difference.
*/
uint8_t shard = (uint8_t)atomic_fetch_add_u(
&arena->binshard_next, 1, ATOMIC_RELAXED);
tsd_binshards_t *bins = tsd_binshardsp_get(tsd);
for (unsigned i = 0; i < SC_NBINS; i++) {
assert(bin_infos[i].n_shards > 0
&& bin_infos[i].n_shards <= BIN_SHARDS_MAX);
bins->binshard[i] = shard % bin_infos[i].n_shards;
}
}
}
void
arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
assert(oldarena != NULL);
assert(newarena != NULL);
arena_nthreads_dec(oldarena, false);
arena_nthreads_inc(newarena, false);
tsd_arena_set(tsd, newarena);
if (arena_nthreads_get(oldarena, false) == 0
&& !background_thread_enabled()) {
/*
* Purge if the old arena has no associated threads anymore and
* no background threads.
*/
arena_decay(tsd_tsdn(tsd), oldarena,
/* is_background_thread */ false, /* all */ true);
}
}
static void
arena_unbind(tsd_t *tsd, unsigned ind, bool internal) {
arena_t *arena;
arena = arena_get(tsd_tsdn(tsd), ind, false);
arena_nthreads_dec(arena, internal);
if (internal) {
tsd_iarena_set(tsd, NULL);
} else {
tsd_arena_set(tsd, NULL);
}
}
/* Slow path, called only by arena_choose(). */
arena_t *
arena_choose_hard(tsd_t *tsd, bool internal) {
arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL);
if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)) {
unsigned choose = percpu_arena_choose();
ret = arena_get(tsd_tsdn(tsd), choose, true);
assert(ret != NULL);
arena_bind(tsd, arena_ind_get(ret), false);
arena_bind(tsd, arena_ind_get(ret), true);
return ret;
}
if (narenas_auto > 1) {
unsigned i, j, choose[2], first_null;
bool is_new_arena[2];
/*
* Determine binding for both non-internal and internal
* allocation.
*
* choose[0]: For application allocation.
* choose[1]: For internal metadata allocation.
*/
for (j = 0; j < 2; j++) {
choose[j] = 0;
is_new_arena[j] = false;
}
first_null = narenas_auto;
malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock);
assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL);
for (i = 1; i < narenas_auto; i++) {
if (arena_get(tsd_tsdn(tsd), i, false) != NULL) {
/*
* Choose the first arena that has the lowest
* number of threads assigned to it.
*/
for (j = 0; j < 2; j++) {
if (arena_nthreads_get(
arena_get(
tsd_tsdn(tsd), i, false),
!!j)
< arena_nthreads_get(
arena_get(tsd_tsdn(tsd),
choose[j], false),
!!j)) {
choose[j] = i;
}
}
} else if (first_null == narenas_auto) {
/*
* Record the index of the first uninitialized
* arena, in case all extant arenas are in use.
*
* NB: It is possible for there to be
* discontinuities in terms of initialized
* versus uninitialized arenas, due to the
* "thread.arena" mallctl.
*/
first_null = i;
}
}
for (j = 0; j < 2; j++) {
if (arena_nthreads_get(
arena_get(tsd_tsdn(tsd), choose[j], false), !!j)
== 0
|| first_null == narenas_auto) {
/*
* Use an unloaded arena, or the least loaded
* arena if all arenas are already initialized.
*/
if (!!j == internal) {
ret = arena_get(
tsd_tsdn(tsd), choose[j], false);
}
} else {
arena_t *arena;
/* Initialize a new arena. */
choose[j] = first_null;
arena = arena_init_locked(tsd_tsdn(tsd),
choose[j], &arena_config_default);
if (arena == NULL) {
malloc_mutex_unlock(
tsd_tsdn(tsd), &arenas_lock);
return NULL;
}
is_new_arena[j] = true;
if (!!j == internal) {
ret = arena;
}
}
arena_bind(tsd, choose[j], !!j);
}
malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock);
for (j = 0; j < 2; j++) {
if (is_new_arena[j]) {
assert(choose[j] > 0);
arena_new_create_background_thread(
tsd_tsdn(tsd), choose[j]);
}
}
} else {
ret = arena_get(tsd_tsdn(tsd), 0, false);
arena_bind(tsd, 0, false);
arena_bind(tsd, 0, true);
}
return ret;
}
void
iarena_cleanup(tsd_t *tsd) {
arena_t *iarena;
iarena = tsd_iarena_get(tsd);
if (iarena != NULL) {
arena_unbind(tsd, arena_ind_get(iarena), true);
}
}
void
arena_cleanup(tsd_t *tsd) {
arena_t *arena;
arena = tsd_arena_get(tsd);
if (arena != NULL) {
arena_unbind(tsd, arena_ind_get(arena), false);
}
}

Some files were not shown because too many files have changed in this diff Show More