Extract fork orchestration into jemalloc_fork module

Makefile.in

@@ -95,6 +95,7 @@ LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix)
 BINS := $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh $(objroot)bin/jeprof
 C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h
 C_SRCS := $(srcroot)src/jemalloc.c \
+	$(srcroot)src/jemalloc_fork.c \
 	$(srcroot)src/jemalloc_init.c \
 	$(srcroot)src/arena.c \
 	$(srcroot)src/arenas_management.c \

include/jemalloc/internal/jemalloc_fork.h

@@ -0,0 +1,8 @@
+#ifndef JEMALLOC_INTERNAL_JEMALLOC_FORK_H
+#define JEMALLOC_INTERNAL_JEMALLOC_FORK_H
+
+void jemalloc_prefork(void);
+void jemalloc_postfork_parent(void);
+void jemalloc_postfork_child(void);
+
+#endif /* JEMALLOC_INTERNAL_JEMALLOC_FORK_H */

include/jemalloc/internal/jemalloc_internal_externs.h

@@ -57,9 +57,6 @@ void    *bootstrap_malloc(size_t size);
 void    *bootstrap_calloc(size_t num, size_t size);
 void     bootstrap_free(void *ptr);
 size_t   batch_alloc(void **ptrs, size_t num, size_t size, int flags);
-void     jemalloc_prefork(void);
-void     jemalloc_postfork_parent(void);
-void     jemalloc_postfork_child(void);
 void     sdallocx_default(void *ptr, size_t size, int flags);
 void     free_default(void *ptr);
 void    *malloc_default(size_t size);

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj

@@ -69,6 +69,7 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters

@@ -208,6 +208,9 @@
     <ClCompile Include="..\..\..\..\src\arenas_management.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj

@@ -69,6 +69,7 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters

@@ -208,6 +208,9 @@
     <ClCompile Include="..\..\..\..\src\arenas_management.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj

@@ -69,6 +69,7 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters

@@ -208,6 +208,9 @@
     <ClCompile Include="..\..\..\..\src\arenas_management.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj

@@ -69,6 +69,7 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters

@@ -208,6 +208,9 @@
     <ClCompile Include="..\..\..\..\src\arenas_management.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
       <Filter>Source Files</Filter>
     </ClCompile>

src/jemalloc.c

@@ -2273,180 +2273,3 @@ label_done:
  * End non-standard functions.
  */
 /******************************************************************************/
-/*
- * The following functions are used by threading libraries for protection of
- * malloc during fork().
- */
-
-/*
- * If an application creates a thread before doing any allocation in the main
- * thread, then calls fork(2) in the main thread followed by memory allocation
- * in the child process, a race can occur that results in deadlock within the
- * child: the main thread may have forked while the created thread had
- * partially initialized the allocator.  Ordinarily jemalloc prevents
- * fork/malloc races via the following functions it registers during
- * initialization using pthread_atfork(), but of course that does no good if
- * the allocator isn't fully initialized at fork time.  The following library
- * constructor is a partial solution to this problem.  It may still be possible
- * to trigger the deadlock described above, but doing so would involve forking
- * via a library constructor that runs before jemalloc's runs.
- */
-#ifndef JEMALLOC_JET
-JEMALLOC_ATTR(constructor)
-static void
-jemalloc_constructor(void) {
-	malloc_init();
-}
-#endif
-
-#ifndef JEMALLOC_MUTEX_INIT_CB
-void
-jemalloc_prefork(void)
-#else
-JEMALLOC_EXPORT void
-_malloc_prefork(void)
-#endif
-{
-	tsd_t   *tsd;
-	unsigned i, j, narenas;
-	arena_t *arena;
-
-#ifdef JEMALLOC_MUTEX_INIT_CB
-	if (!malloc_initialized()) {
-		return;
-	}
-#endif
-	assert(malloc_initialized());
-
-	tsd = tsd_fetch();
-
-	narenas = narenas_total_get();
-
-	witness_prefork(tsd_witness_tsdp_get(tsd));
-	/* Acquire all mutexes in a safe order. */
-	ctl_prefork(tsd_tsdn(tsd));
-	tcache_prefork(tsd_tsdn(tsd));
-	arenas_management_prefork(tsd_tsdn(tsd));
-	if (have_background_thread) {
-		background_thread_prefork0(tsd_tsdn(tsd));
-	}
-	prof_prefork0(tsd_tsdn(tsd));
-	if (have_background_thread) {
-		background_thread_prefork1(tsd_tsdn(tsd));
-	}
-	/* Break arena prefork into stages to preserve lock order. */
-	for (i = 0; i < 9; i++) {
-		for (j = 0; j < narenas; j++) {
-			if ((arena = arena_get(tsd_tsdn(tsd), j, false))
-			    != NULL) {
-				switch (i) {
-				case 0:
-					arena_prefork0(tsd_tsdn(tsd), arena);
-					break;
-				case 1:
-					arena_prefork1(tsd_tsdn(tsd), arena);
-					break;
-				case 2:
-					arena_prefork2(tsd_tsdn(tsd), arena);
-					break;
-				case 3:
-					arena_prefork3(tsd_tsdn(tsd), arena);
-					break;
-				case 4:
-					arena_prefork4(tsd_tsdn(tsd), arena);
-					break;
-				case 5:
-					arena_prefork5(tsd_tsdn(tsd), arena);
-					break;
-				case 6:
-					arena_prefork6(tsd_tsdn(tsd), arena);
-					break;
-				case 7:
-					arena_prefork7(tsd_tsdn(tsd), arena);
-					break;
-				case 8:
-					arena_prefork8(tsd_tsdn(tsd), arena);
-					break;
-				default:
-					not_reached();
-				}
-			}
-		}
-	}
-	prof_prefork1(tsd_tsdn(tsd));
-	stats_prefork(tsd_tsdn(tsd));
-	tsd_prefork(tsd);
-}
-
-#ifndef JEMALLOC_MUTEX_INIT_CB
-void
-jemalloc_postfork_parent(void)
-#else
-JEMALLOC_EXPORT void
-_malloc_postfork(void)
-#endif
-{
-	tsd_t   *tsd;
-	unsigned i, narenas;
-
-#ifdef JEMALLOC_MUTEX_INIT_CB
-	if (!malloc_initialized()) {
-		return;
-	}
-#endif
-	assert(malloc_initialized());
-
-	tsd = tsd_fetch();
-
-	tsd_postfork_parent(tsd);
-
-	witness_postfork_parent(tsd_witness_tsdp_get(tsd));
-	/* Release all mutexes, now that fork() has completed. */
-	stats_postfork_parent(tsd_tsdn(tsd));
-	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
-		arena_t *arena;
-
-		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
-			arena_postfork_parent(tsd_tsdn(tsd), arena);
-		}
-	}
-	prof_postfork_parent(tsd_tsdn(tsd));
-	if (have_background_thread) {
-		background_thread_postfork_parent(tsd_tsdn(tsd));
-	}
-	arenas_management_postfork_parent(tsd_tsdn(tsd));
-	tcache_postfork_parent(tsd_tsdn(tsd));
-	ctl_postfork_parent(tsd_tsdn(tsd));
-}
-
-void
-jemalloc_postfork_child(void) {
-	tsd_t   *tsd;
-	unsigned i, narenas;
-
-	assert(malloc_initialized());
-
-	tsd = tsd_fetch();
-
-	tsd_postfork_child(tsd);
-
-	witness_postfork_child(tsd_witness_tsdp_get(tsd));
-	/* Release all mutexes, now that fork() has completed. */
-	stats_postfork_child(tsd_tsdn(tsd));
-	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
-		arena_t *arena;
-
-		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
-			arena_postfork_child(tsd_tsdn(tsd), arena);
-		}
-	}
-	prof_postfork_child(tsd_tsdn(tsd));
-	if (have_background_thread) {
-		background_thread_postfork_child(tsd_tsdn(tsd));
-	}
-	arenas_management_postfork_child(tsd_tsdn(tsd));
-	tcache_postfork_child(tsd_tsdn(tsd));
-	ctl_postfork_child(tsd_tsdn(tsd));
-}
-
-/******************************************************************************/

src/jemalloc_fork.c

@@ -0,0 +1,163 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/arenas_management.h"
+#include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/jemalloc_fork.h"
+#include "jemalloc/internal/jemalloc_init.h"
+
+/******************************************************************************/
+/*
+ * The following functions are used by threading libraries for protection of
+ * malloc during fork().
+ */
+
+#ifndef JEMALLOC_MUTEX_INIT_CB
+void
+jemalloc_prefork(void)
+#else
+JEMALLOC_EXPORT void
+_malloc_prefork(void)
+#endif
+{
+	tsd_t   *tsd;
+	unsigned i, j, narenas;
+	arena_t *arena;
+
+#ifdef JEMALLOC_MUTEX_INIT_CB
+	if (!malloc_initialized()) {
+		return;
+	}
+#endif
+	assert(malloc_initialized());
+
+	tsd = tsd_fetch();
+
+	narenas = narenas_total_get();
+
+	witness_prefork(tsd_witness_tsdp_get(tsd));
+	/* Acquire all mutexes in a safe order. */
+	ctl_prefork(tsd_tsdn(tsd));
+	tcache_prefork(tsd_tsdn(tsd));
+	arenas_management_prefork(tsd_tsdn(tsd));
+	if (have_background_thread) {
+		background_thread_prefork0(tsd_tsdn(tsd));
+	}
+	prof_prefork0(tsd_tsdn(tsd));
+	if (have_background_thread) {
+		background_thread_prefork1(tsd_tsdn(tsd));
+	}
+	/* Break arena prefork into stages to preserve lock order. */
+	for (i = 0; i < 9; i++) {
+		for (j = 0; j < narenas; j++) {
+			if ((arena = arena_get(tsd_tsdn(tsd), j, false))
+			    != NULL) {
+				switch (i) {
+				case 0:
+					arena_prefork0(tsd_tsdn(tsd), arena);
+					break;
+				case 1:
+					arena_prefork1(tsd_tsdn(tsd), arena);
+					break;
+				case 2:
+					arena_prefork2(tsd_tsdn(tsd), arena);
+					break;
+				case 3:
+					arena_prefork3(tsd_tsdn(tsd), arena);
+					break;
+				case 4:
+					arena_prefork4(tsd_tsdn(tsd), arena);
+					break;
+				case 5:
+					arena_prefork5(tsd_tsdn(tsd), arena);
+					break;
+				case 6:
+					arena_prefork6(tsd_tsdn(tsd), arena);
+					break;
+				case 7:
+					arena_prefork7(tsd_tsdn(tsd), arena);
+					break;
+				case 8:
+					arena_prefork8(tsd_tsdn(tsd), arena);
+					break;
+				default:
+					not_reached();
+				}
+			}
+		}
+	}
+	prof_prefork1(tsd_tsdn(tsd));
+	stats_prefork(tsd_tsdn(tsd));
+	tsd_prefork(tsd);
+}
+
+#ifndef JEMALLOC_MUTEX_INIT_CB
+void
+jemalloc_postfork_parent(void)
+#else
+JEMALLOC_EXPORT void
+_malloc_postfork(void)
+#endif
+{
+	tsd_t   *tsd;
+	unsigned i, narenas;
+
+#ifdef JEMALLOC_MUTEX_INIT_CB
+	if (!malloc_initialized()) {
+		return;
+	}
+#endif
+	assert(malloc_initialized());
+
+	tsd = tsd_fetch();
+
+	tsd_postfork_parent(tsd);
+
+	witness_postfork_parent(tsd_witness_tsdp_get(tsd));
+	/* Release all mutexes, now that fork() has completed. */
+	stats_postfork_parent(tsd_tsdn(tsd));
+	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
+		arena_t *arena;
+
+		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
+			arena_postfork_parent(tsd_tsdn(tsd), arena);
+		}
+	}
+	prof_postfork_parent(tsd_tsdn(tsd));
+	if (have_background_thread) {
+		background_thread_postfork_parent(tsd_tsdn(tsd));
+	}
+	arenas_management_postfork_parent(tsd_tsdn(tsd));
+	tcache_postfork_parent(tsd_tsdn(tsd));
+	ctl_postfork_parent(tsd_tsdn(tsd));
+}
+
+void
+jemalloc_postfork_child(void) {
+	tsd_t   *tsd;
+	unsigned i, narenas;
+
+	assert(malloc_initialized());
+
+	tsd = tsd_fetch();
+
+	tsd_postfork_child(tsd);
+
+	witness_postfork_child(tsd_witness_tsdp_get(tsd));
+	/* Release all mutexes, now that fork() has completed. */
+	stats_postfork_child(tsd_tsdn(tsd));
+	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
+		arena_t *arena;
+
+		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
+			arena_postfork_child(tsd_tsdn(tsd), arena);
+		}
+	}
+	prof_postfork_child(tsd_tsdn(tsd));
+	if (have_background_thread) {
+		background_thread_postfork_child(tsd_tsdn(tsd));
+	}
+	arenas_management_postfork_child(tsd_tsdn(tsd));
+	tcache_postfork_child(tsd_tsdn(tsd));
+	ctl_postfork_child(tsd_tsdn(tsd));
+}

src/jemalloc_init.c

@@ -8,6 +8,7 @@
 #include "jemalloc/internal/extent_dss.h"
 #include "jemalloc/internal/extent_mmap.h"
 #include "jemalloc/internal/hook.h"
+#include "jemalloc/internal/jemalloc_fork.h"
 #include "jemalloc/internal/jemalloc_init.h"
 #include "jemalloc/internal/malloc_io.h"
 #include "jemalloc/internal/mutex.h"
@@ -685,3 +686,24 @@ malloc_init_hard(void) {
 #undef UNLOCK_RETURN
 	return false;
 }
+
+/*
+ * If an application creates a thread before doing any allocation in the main
+ * thread, then calls fork(2) in the main thread followed by memory allocation
+ * in the child process, a race can occur that results in deadlock within the
+ * child: the main thread may have forked while the created thread had
+ * partially initialized the allocator.  Ordinarily jemalloc prevents
+ * fork/malloc races via the following functions it registers during
+ * initialization using pthread_atfork(), but of course that does no good if
+ * the allocator isn't fully initialized at fork time.  The following library
+ * constructor is a partial solution to this problem.  It may still be possible
+ * to trigger the deadlock described above, but doing so would involve forking
+ * via a library constructor that runs before jemalloc's runs.
+ */
+#ifndef JEMALLOC_JET
+JEMALLOC_ATTR(constructor)
+static void
+jemalloc_constructor(void) {
+	malloc_init();
+}
+#endif

src/zone.c

@@ -2,6 +2,7 @@
 #include "jemalloc/internal/jemalloc_internal_includes.h"
 
 #include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/jemalloc_fork.h"
 
 #ifndef JEMALLOC_ZONE
 #	error "This source file is for zones on Darwin (OS X)."