De-export test-only helpers via JET_EXTERN / #ifdef JEMALLOC_JET; declare in unit tests, not headers

include/jemalloc/internal/arena.h

@@ -240,7 +240,6 @@ void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
 void  arena_prof_promote(
      tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
 size_t arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr);
-void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
 
 void  arena_dalloc_small(tsdn_t *tsdn, void *ptr);
 void  arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
@@ -274,8 +273,6 @@ void   arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena,
        cache_bin_array_descriptor_t *desc);
 void   arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
        cache_bin_array_descriptor_t *desc);
-void   arena_cache_bin_array_postfork_child(arena_t *arena,
-       cache_bin_array_descriptor_t *desc_or_null);
 void   arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork0(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork1(tsdn_t *tsdn, arena_t *arena);

include/jemalloc/internal/base.h

@@ -106,7 +106,6 @@ base_t *base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks,
     bool metadata_use_hooks);
 void    base_delete(tsdn_t *tsdn, base_t *base);
 ehooks_t       *base_ehooks_get(base_t *base);
-ehooks_t       *base_ehooks_get_for_metadata(base_t *base);
 extent_hooks_t *base_extent_hooks_set(
     base_t *base, extent_hooks_t *extent_hooks);
 void    *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment);

include/jemalloc/internal/bin.h

@@ -69,23 +69,13 @@ void bin_postfork_parent(tsdn_t *tsdn, bin_t *bin);
 void bin_postfork_child(tsdn_t *tsdn, bin_t *bin);
 
 /* Slab region allocation. */
-void *bin_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info);
 void  bin_slab_reg_alloc_batch(
      edata_t *slab, const bin_info_t *bin_info, unsigned cnt, void **ptrs);
 
 /* Slab list management. */
-void     bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab);
-void     bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab);
-edata_t *bin_slabs_nonfull_tryget(bin_t *bin);
-void     bin_slabs_full_insert(bool is_auto, bin_t *bin, edata_t *slab);
 void     bin_slabs_full_remove(bool is_auto, bin_t *bin, edata_t *slab);
 
-/* Slab association / demotion. */
-void bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin);
-void bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin);
-
 /* Deallocation helpers (called under bin lock). */
-void bin_dalloc_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin);
 void bin_dalloc_locked_handle_newly_empty(
     tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin);
 void bin_dalloc_locked_handle_newly_nonempty(

include/jemalloc/internal/bitmap.h

@@ -176,9 +176,7 @@ typedef struct bitmap_info_s {
 #endif /* BITMAP_USE_TREE */
 } bitmap_info_t;
 
-void   bitmap_info_init(bitmap_info_t *binfo, size_t nbits);
 void   bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill);
-size_t bitmap_size(const bitmap_info_t *binfo);
 
 static inline bool
 bitmap_full(const bitmap_t *bitmap, const bitmap_info_t *binfo) {

include/jemalloc/internal/buf_writer.h

@@ -25,7 +25,6 @@ typedef struct {
 
 bool       buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer,
           write_cb_t *write_cb, void *cbopaque, char *buf, size_t buf_len);
-void       buf_writer_flush(buf_writer_t *buf_writer);
 write_cb_t buf_writer_cb;
 void       buf_writer_terminate(tsdn_t *tsdn, buf_writer_t *buf_writer);
 

include/jemalloc/internal/ckh.h

@@ -93,10 +93,4 @@ bool ckh_remove(
     tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, void **data);
 bool ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data);
 
-/* Some useful hash and comparison functions for strings and pointers. */
-void ckh_string_hash(const void *key, size_t r_hash[2]);
-bool ckh_string_keycomp(const void *k1, const void *k2);
-void ckh_pointer_hash(const void *key, size_t r_hash[2]);
-bool ckh_pointer_keycomp(const void *k1, const void *k2);
-
 #endif /* JEMALLOC_INTERNAL_CKH_H */

include/jemalloc/internal/ehooks.h

@@ -57,13 +57,6 @@ bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length);
 bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length);
 #endif
 bool ehooks_default_split_impl(void);
-/*
- * Merge is the only default extent hook we declare -- see the comment in
- * ehooks_merge.
- */
-bool ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a,
-    size_t size_a, void *addr_b, size_t size_b, bool committed,
-    unsigned arena_ind);
 bool ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b);
 void ehooks_default_zero_impl(void *addr, size_t size);
 void ehooks_default_guard_impl(void *guard1, void *guard2);

include/jemalloc/internal/hpa.h

@@ -190,7 +190,6 @@ struct hpa_shard_s {
 	nstime_t last_time_work_attempted;
 };
 
-bool hpa_hugepage_size_exceeds_limit(void);
 /*
  * Whether or not the HPA can be used given the current configuration.  This
  * is not necessarily a guarantee that it backs its allocations by hugepages,
@@ -204,11 +203,6 @@ bool hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
 edata_t *hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
     size_t alignment, bool zero, bool guarded, bool frequent_reuse,
     bool *deferred_work_generated);
-bool hpa_expand(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
-    size_t old_size, size_t new_size, bool zero,
-    bool *deferred_work_generated);
-bool hpa_shrink(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
-    size_t old_size, size_t new_size, bool *deferred_work_generated);
 void hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
     bool *deferred_work_generated);
 uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard);

include/jemalloc/internal/hpdata.h

@@ -425,11 +425,6 @@ hpdata_full(const hpdata_t *hpdata) {
 
 void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age, bool is_huge);
 
-/*
- * Given an hpdata which can serve an allocation request, pick and reserve an
- * offset within that allocation.
- */
-void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
 void  hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz);
 
 /*

include/jemalloc/internal/nstime.h

@@ -28,7 +28,6 @@ static const nstime_t nstime_zero = NSTIME_ZERO_INITIALIZER;
 void     nstime_init(nstime_t *time, uint64_t ns);
 void     nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec);
 uint64_t nstime_ns(const nstime_t *time);
-uint64_t nstime_ms(const nstime_t *time);
 uint64_t nstime_sec(const nstime_t *time);
 uint64_t nstime_nsec(const nstime_t *time);
 void     nstime_copy(nstime_t *time, const nstime_t *source);
@@ -36,11 +35,9 @@ int      nstime_compare(const nstime_t *a, const nstime_t *b);
 void     nstime_add(nstime_t *time, const nstime_t *addend);
 void     nstime_iadd(nstime_t *time, uint64_t addend);
 void     nstime_subtract(nstime_t *time, const nstime_t *subtrahend);
-void     nstime_isubtract(nstime_t *time, uint64_t subtrahend);
 void     nstime_imultiply(nstime_t *time, uint64_t multiplier);
 void     nstime_idivide(nstime_t *time, uint64_t divisor);
 uint64_t nstime_divide(const nstime_t *time, const nstime_t *divisor);
-uint64_t nstime_ns_between(const nstime_t *earlier, const nstime_t *later);
 uint64_t nstime_ms_between(const nstime_t *earlier, const nstime_t *later);
 uint64_t nstime_ns_since(const nstime_t *past);
 uint64_t nstime_ms_since(const nstime_t *past);

include/jemalloc/internal/pa.h

@@ -220,8 +220,6 @@ void pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard);
 void pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard);
 
 size_t pa_shard_nactive(const pa_shard_t *shard);
-size_t pa_shard_ndirty(const pa_shard_t *shard);
-size_t pa_shard_nmuzzy(const pa_shard_t *shard);
 
 void pa_shard_basic_stats_merge(
     const pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);

include/jemalloc/internal/pac.h

@@ -234,7 +234,6 @@ bool    pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
        ssize_t decay_ms, pac_purge_eagerness_t eagerness);
 ssize_t pac_decay_ms_get(pac_t *pac, extent_state_t state);
 
-void pac_reset(tsdn_t *tsdn, pac_t *pac);
 void pac_destroy(tsdn_t *tsdn, pac_t *pac);
 
 #endif /* JEMALLOC_INTERNAL_PAC_H */

include/jemalloc/internal/prof.h

@@ -419,7 +419,6 @@ void        prof_prefork1(tsdn_t *tsdn);
 void        prof_postfork_parent(tsdn_t *tsdn);
 void        prof_postfork_child(tsdn_t *tsdn);
 
-uint64_t prof_sample_new_event_wait(tsd_t *tsd);
 uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd);
 
 extern te_base_cb_t prof_sample_te_handler;

include/jemalloc/internal/prof_data.h

@@ -29,9 +29,4 @@ void          prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata);
 void          prof_reset(tsd_t *tsd, size_t lg_sample);
 void          prof_tctx_try_destroy(tsd_t *tsd, prof_tctx_t *tctx);
 
-/* Used in unit tests. */
-size_t prof_tdata_count(void);
-size_t prof_bt_count(void);
-void   prof_cnt_all(prof_cnt_t *cnt_all);
-
 #endif /* JEMALLOC_INTERNAL_PROF_DATA_H */

include/jemalloc/internal/prof_log.h

@@ -9,14 +9,6 @@ extern malloc_mutex_t log_mtx;
 void prof_try_log(tsd_t *tsd, size_t usize, prof_info_t *prof_info);
 bool prof_log_init(tsd_t *tsdn);
 
-/* Used in unit tests. */
-size_t prof_log_bt_count(void);
-size_t prof_log_alloc_count(void);
-size_t prof_log_thr_count(void);
-bool   prof_log_is_logging(void);
-bool   prof_log_rep_check(void);
-void   prof_log_dummy_set(bool new_value);
-
 bool prof_log_start(tsdn_t *tsdn, const char *filename);
 bool prof_log_stop(tsdn_t *tsdn);
 

include/jemalloc/internal/prof_recent.h

@@ -17,8 +17,6 @@ void edata_prof_recent_alloc_init(edata_t *edata);
 /* Used in unit tests. */
 typedef ql_head(prof_recent_t) prof_recent_list_t;
 extern prof_recent_list_t prof_recent_alloc_list;
-edata_t *prof_recent_alloc_edata_get_no_lock_test(const prof_recent_t *node);
-prof_recent_t *edata_prof_recent_alloc_get_no_lock_test(const edata_t *edata);
 
 ssize_t prof_recent_alloc_max_ctl_read(void);
 ssize_t prof_recent_alloc_max_ctl_write(tsd_t *tsd, ssize_t max);

include/jemalloc/internal/sc.h

@@ -368,7 +368,6 @@ struct sc_data_s {
 	sc_t sc[SC_NSIZES];
 };
 
-size_t reg_size_compute(int lg_base, int lg_delta, int ndelta);
 void   sc_data_init(sc_data_t *data);
 /*
  * Updates slab sizes in [begin, end] to be pgs pages in length, if possible.

src/arena.c

@@ -294,7 +294,7 @@ arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
  * from scratch (descriptors held by other threads at fork time are gone)
  * without locking.
  */
-void
+static void
 arena_cache_bin_array_postfork_child(arena_t *arena,
     cache_bin_array_descriptor_t *desc_or_null) {
 	cassert(config_stats);
@@ -652,7 +652,7 @@ arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena) {
 	pa_shard_do_deferred_work(tsdn, &arena->pa_shard);
 }
 
-void
+static void
 arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab) {
 	bool deferred_work_generated = false;
 	pa_dalloc(tsdn, &arena->pa_shard, slab, &deferred_work_generated);

src/base.c

@@ -8,6 +8,8 @@
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/sz.h"
 
+JET_EXTERN ehooks_t *base_ehooks_get_for_metadata(base_t *base);
+
 /*
  * In auto mode, arenas switch to huge pages for the base allocator on the
  * second base block.  a0 switches to thp on the 5th block (after 20 megabytes
@@ -545,7 +547,7 @@ base_ehooks_get(base_t *base) {
 	return &base->ehooks;
 }
 
-ehooks_t *
+JET_EXTERN ehooks_t *
 base_ehooks_get_for_metadata(base_t *base) {
 	return &base->ehooks_base;
 }

src/bin.c

@@ -69,7 +69,7 @@ bin_postfork_child(tsdn_t *tsdn, bin_t *bin) {
 	malloc_mutex_postfork_child(tsdn, &bin->lock);
 }
 
-void *
+JET_EXTERN void *
 bin_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info) {
 	void        *ret;
 	slab_data_t *slab_data = edata_slab_data_get(slab);
@@ -134,7 +134,7 @@ bin_slab_reg_alloc_batch(
 	edata_nfree_sub(slab, cnt);
 }
 
-void
+JET_EXTERN void
 bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab) {
 	assert(edata_nfree_get(slab) > 0);
 	edata_heap_insert(&bin->slabs_nonfull, slab);
@@ -143,7 +143,7 @@ bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab) {
 	}
 }
 
-void
+JET_EXTERN void
 bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab) {
 	edata_heap_remove(&bin->slabs_nonfull, slab);
 	if (config_stats) {
@@ -151,7 +151,7 @@ bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab) {
 	}
 }
 
-edata_t *
+JET_EXTERN edata_t *
 bin_slabs_nonfull_tryget(bin_t *bin) {
 	edata_t *slab = edata_heap_remove_first(&bin->slabs_nonfull);
 	if (slab == NULL) {
@@ -164,7 +164,7 @@ bin_slabs_nonfull_tryget(bin_t *bin) {
 	return slab;
 }
 
-void
+JET_EXTERN void
 bin_slabs_full_insert(bool is_auto, bin_t *bin, edata_t *slab) {
 	assert(edata_nfree_get(slab) == 0);
 	/*
@@ -186,7 +186,7 @@ bin_slabs_full_remove(bool is_auto, bin_t *bin, edata_t *slab) {
 	edata_list_active_remove(&bin->slabs_full, slab);
 }
 
-void
+JET_EXTERN void
 bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin) {
 	/* Dissociate slab from bin. */
 	if (slab == bin->slabcur) {
@@ -208,7 +208,7 @@ bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin) {
 	}
 }
 
-void
+JET_EXTERN void
 bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin) {
 	assert(edata_nfree_get(slab) > 0);
 
@@ -234,7 +234,7 @@ bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin) {
 	}
 }
 
-void
+JET_EXTERN void
 bin_dalloc_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin) {
 	malloc_mutex_assert_owner(tsdn, &bin->lock);
 

src/bitmap.c

@@ -5,8 +5,11 @@
 
 /******************************************************************************/
 
+JET_EXTERN size_t bitmap_size(const bitmap_info_t *binfo);
+
 #ifdef BITMAP_USE_TREE
 
+#ifdef JEMALLOC_JET
 void
 bitmap_info_init(bitmap_info_t *binfo, size_t nbits) {
 	unsigned i;
@@ -34,6 +37,7 @@ bitmap_info_init(bitmap_info_t *binfo, size_t nbits) {
 	binfo->nlevels = i;
 	binfo->nbits = nbits;
 }
+#endif
 
 static size_t
 bitmap_info_ngroups(const bitmap_info_t *binfo) {
@@ -82,6 +86,7 @@ bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill) {
 
 #else /* BITMAP_USE_TREE */
 
+#ifdef JEMALLOC_JET
 void
 bitmap_info_init(bitmap_info_t *binfo, size_t nbits) {
 	assert(nbits > 0);
@@ -90,6 +95,7 @@ bitmap_info_init(bitmap_info_t *binfo, size_t nbits) {
 	binfo->ngroups = BITMAP_BITS2GROUPS(nbits);
 	binfo->nbits = nbits;
 }
+#endif
 
 static size_t
 bitmap_info_ngroups(const bitmap_info_t *binfo) {
@@ -115,7 +121,7 @@ bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill) {
 
 #endif /* BITMAP_USE_TREE */
 
-size_t
+JET_EXTERN size_t
 bitmap_size(const bitmap_info_t *binfo) {
 	return (bitmap_info_ngroups(binfo) << LG_SIZEOF_BITMAP);
 }

src/buf_writer.c

@@ -68,7 +68,7 @@ buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer, write_cb_t *write_cb,
 	return buf_writer->buf == NULL;
 }
 
-void
+JET_EXTERN void
 buf_writer_flush(buf_writer_t *buf_writer) {
 	buf_writer_assert(buf_writer);
 	if (buf_writer->buf == NULL) {

src/ckh.c

@@ -541,6 +541,7 @@ ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data) {
 	return true;
 }
 
+#ifdef JEMALLOC_JET
 void
 ckh_string_hash(const void *key, size_t r_hash[2]) {
 	hash(key, strlen((const char *)key), 0x94122f33U, r_hash);
@@ -570,3 +571,4 @@ bool
 ckh_pointer_keycomp(const void *k1, const void *k2) {
 	return (k1 == k2);
 }
+#endif

src/ctl.c

@@ -26,6 +26,29 @@
 #include "jemalloc/internal/util.h"
 #include "jemalloc/internal/witness.h"
 
+/******************************************************************************/
+/*
+ * File layout.  Each section below is introduced by a matching banner comment
+ * lower in the file; search the section text to jump to it.
+ *
+ *   Data
+ *   Helpers for named and indexed nodes
+ *   Function prototypes for non-inline static functions
+ *   mallctl tree
+ *   Arena stats accumulation
+ *   ctl engine: arena indexing, refresh/init, lookup, boot/fork
+ *   *_ctl() functions          (shared helpers + read-only generators)
+ *   Top-level handlers          (version, epoch, background_thread)
+ *   config.* handlers
+ *   opt.* handlers
+ *   thread.* handlers
+ *   tcache.* handlers
+ *   arena.<i>.* handlers
+ *   arenas.* handlers
+ *   prof.* and experimental.hooks.* handlers
+ *   stats.* handlers          (+ index callbacks and experimental/prof.stats tail)
+ */
+
 /******************************************************************************/
 /* Data. */
 
@@ -952,6 +975,7 @@ static const ctl_named_node_t super_root_node[] = {
 #undef INDEX
 
 /******************************************************************************/
+/* Arena stats accumulation. */
 
 /*
  * Sets *dst + *src non-atomically.  This is safe, since everything is
@@ -970,6 +994,7 @@ ctl_accum_atomic_zu(atomic_zu_t *dst, atomic_zu_t *src) {
 }
 
 /******************************************************************************/
+/* ctl engine: arena indexing, refresh/init, lookup, boot/fork. */
 
 /*
  * Historical compatibility for treating arena.<narenas> as the merged
@@ -2028,6 +2053,7 @@ ctl_mib_unsigned(unsigned *dst, const size_t *mib, size_t mib_index) {
 	}
 
 /******************************************************************************/
+/* Top-level handlers: version, epoch, background_thread. */
 
 CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *)
 
@@ -2151,6 +2177,7 @@ label_return:
 }
 
 /******************************************************************************/
+/* config.* handlers. */
 
 CTL_RO_CONFIG_GEN(config_cache_oblivious, bool)
 CTL_RO_CONFIG_GEN(config_debug, bool)
@@ -2168,6 +2195,7 @@ CTL_RO_CONFIG_GEN(config_utrace, bool)
 CTL_RO_CONFIG_GEN(config_xmalloc, bool)
 
 /******************************************************************************/
+/* opt.* handlers. */
 
 CTL_RO_NL_GEN(opt_abort, opt_abort, bool)
 CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool)
@@ -2287,6 +2315,7 @@ CTL_RO_NL_CGEN(
     je_malloc_conf, opt_malloc_conf_global_var, je_malloc_conf, const char *)
 
 /******************************************************************************/
+/* thread.* handlers. */
 
 static int
 thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
@@ -2575,6 +2604,7 @@ thread_idle_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 }
 
 /******************************************************************************/
+/* tcache.* handlers. */
 
 static int
 tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
@@ -2635,6 +2665,7 @@ tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 }
 
 /******************************************************************************/
+/* arena.<i>.* handlers. */
 
 static int
 arena_i_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
@@ -3160,6 +3191,7 @@ label_return:
 }
 
 /******************************************************************************/
+/* arenas.* handlers. */
 
 static int
 arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
@@ -3318,6 +3350,7 @@ arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 }
 
 /******************************************************************************/
+/* prof.* and experimental.hooks.* handlers. */
 
 static int
 prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
@@ -3653,6 +3686,7 @@ experimental_hooks_thread_event_ctl(tsd_t *tsd, const size_t *mib,
 }
 
 /******************************************************************************/
+/* stats.* handlers (+ index callbacks and experimental/prof.stats tail). */
 
 CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t)
 CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t)

src/ehooks.c

@@ -228,7 +228,7 @@ ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b) {
 	return false;
 }
 
-bool
+static bool
 ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
     void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
 	tsdn_t *tsdn = tsdn_fetch();

src/hpa.c

@@ -14,7 +14,7 @@ const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"};
 bool opt_experimental_hpa_start_huge_if_thp_always = true;
 bool opt_experimental_hpa_enforce_hugify = false;
 
-bool
+JET_EXTERN bool
 hpa_hugepage_size_exceeds_limit(void) {
 	return HUGEPAGE > HUGEPAGE_MAX_EXPECTED_SIZE;
 }
@@ -978,20 +978,6 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
 	return edata;
 }
 
-bool
-hpa_expand(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, size_t old_size,
-    size_t new_size, bool zero, bool *deferred_work_generated) {
-	/* Expand not yet supported. */
-	return true;
-}
-
-bool
-hpa_shrink(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, size_t old_size,
-    size_t new_size, bool *deferred_work_generated) {
-	/* Shrink not yet supported. */
-	return true;
-}
-
 static void
 hpa_dalloc_prepare_unlocked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) {
 	malloc_mutex_assert_not_owner(tsdn, &shard->mtx);

src/hpdata.c

@@ -46,6 +46,7 @@ ph_gen(, hpdata_age_heap, hpdata_t, age_link, hpdata_age_comp)
 	hpdata_assert_consistent(hpdata);
 }
 
+#ifdef JEMALLOC_JET
 void *
 hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) {
 	hpdata_assert_consistent(hpdata);
@@ -138,6 +139,7 @@ hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) {
 	return (
 	    void *)((byte_t *)hpdata_addr_get(hpdata) + (result << LG_PAGE));
 }
+#endif
 
 void
 hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) {

src/nstime.c

@@ -60,11 +60,13 @@ nstime_ns(const nstime_t *time) {
 	return time->ns;
 }
 
+#ifdef JEMALLOC_JET
 uint64_t
 nstime_ms(const nstime_t *time) {
 	nstime_assert_initialized(time);
 	return time->ns / MILLION;
 }
+#endif
 
 uint64_t
 nstime_sec(const nstime_t *time) {
@@ -119,6 +121,7 @@ nstime_subtract(nstime_t *time, const nstime_t *subtrahend) {
 	time->ns -= subtrahend->ns;
 }
 
+#ifdef JEMALLOC_JET
 void
 nstime_isubtract(nstime_t *time, uint64_t subtrahend) {
 	nstime_assert_initialized(time);
@@ -127,6 +130,7 @@ nstime_isubtract(nstime_t *time, uint64_t subtrahend) {
 	/* No initialize operand -- subtraction must be initialized. */
 	time->ns -= subtrahend;
 }
+#endif
 
 void
 nstime_imultiply(nstime_t *time, uint64_t multiplier) {
@@ -158,7 +162,7 @@ nstime_divide(const nstime_t *time, const nstime_t *divisor) {
 	return time->ns / divisor->ns;
 }
 
-uint64_t
+static uint64_t
 nstime_ns_between(const nstime_t *earlier, const nstime_t *later) {
 	nstime_assert_initialized(earlier);
 	nstime_assert_initialized(later);

src/pa_extra.c

@@ -81,7 +81,7 @@ pa_shard_nactive(const pa_shard_t *shard) {
 	return atomic_load_zu(&shard->nactive, ATOMIC_RELAXED);
 }
 
-size_t
+static size_t
 pa_shard_ndirty(const pa_shard_t *shard) {
 	size_t ndirty = ecache_npages_get(&shard->pac.ecache_dirty);
 	if (shard->ever_used_hpa) {
@@ -90,7 +90,7 @@ pa_shard_ndirty(const pa_shard_t *shard) {
 	return ndirty;
 }
 
-size_t
+static size_t
 pa_shard_nmuzzy(const pa_shard_t *shard) {
 	return ecache_npages_get(&shard->pac.ecache_muzzy);
 }

src/pac.c

@@ -747,16 +747,6 @@ pac_decay_ms_get(pac_t *pac, extent_state_t state) {
 	return decay_ms_read(decay);
 }
 
-void
-pac_reset(tsdn_t *tsdn, pac_t *pac) {
-	/*
-	 * No-op for now; purging is still done at the arena-level.  It should
-	 * get moved in here, though.
-	 */
-	(void)tsdn;
-	(void)pac;
-}
-
 void
 pac_destroy(tsdn_t *tsdn, pac_t *pac) {
 	assert(ecache_npages_get(&pac->ecache_dirty) == 0);

src/prof.c

@@ -249,7 +249,7 @@ prof_tctx_create(tsd_t *tsd) {
  * (e.g.
  * -mno-sse) in order for the workaround to be complete.
  */
-uint64_t
+JET_EXTERN uint64_t
 prof_sample_new_event_wait(tsd_t *tsd) {
 #ifdef JEMALLOC_PROF
 	if (lg_prof_sample == 0) {

src/prof_data.c

@@ -404,6 +404,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) {
 	return ret.p;
 }
 
+#ifdef JEMALLOC_JET
 /* Used in unit tests. */
 static prof_tdata_t *
 prof_tdata_count_iter(
@@ -449,6 +450,7 @@ prof_bt_count(void) {
 
 	return bt_count;
 }
+#endif
 
 static void
 prof_thread_name_write_tdata(prof_tdata_t *tdata, const char *thread_name) {
@@ -1105,6 +1107,7 @@ prof_dump_impl(tsd_t *tsd, write_cb_t *prof_dump_write, void *cbopaque,
 	}
 }
 
+#ifdef JEMALLOC_JET
 /* Used in unit tests. */
 void
 prof_cnt_all(prof_cnt_t *cnt_all) {
@@ -1119,6 +1122,7 @@ prof_cnt_all(prof_cnt_t *cnt_all) {
 		prof_gctx_finish(tsd, &gctxs);
 	}
 }
+#endif
 
 void
 prof_bt_hash(const void *key, size_t r_hash[2]) {

src/prof_log.c

@@ -309,6 +309,7 @@ prof_thr_node_keycomp(const void *k1, const void *k2) {
 	return thr_node1->thr_uid == thr_node2->thr_uid;
 }
 
+#ifdef JEMALLOC_JET
 /* Used in unit tests. */
 size_t
 prof_log_bt_count(void) {
@@ -413,6 +414,7 @@ prof_log_dummy_set(bool new_value) {
 	cassert(config_prof);
 	prof_log_dummy = new_value;
 }
+#endif
 
 /* Used as an atexit function to stop logging on exit. */
 static void

src/prof_recent.c

@@ -108,11 +108,13 @@ prof_recent_alloc_edata_get_no_lock(const prof_recent_t *n) {
 	return (edata_t *)atomic_load_p(&n->alloc_edata, ATOMIC_ACQUIRE);
 }
 
+#ifdef JEMALLOC_JET
 edata_t *
 prof_recent_alloc_edata_get_no_lock_test(const prof_recent_t *n) {
 	cassert(config_prof);
 	return prof_recent_alloc_edata_get_no_lock(n);
 }
+#endif
 
 static inline edata_t *
 prof_recent_alloc_edata_get(tsd_t *tsd, const prof_recent_t *n) {
@@ -138,11 +140,13 @@ edata_prof_recent_alloc_get_no_lock(const edata_t *edata) {
 	return edata_prof_recent_alloc_get_dont_call_directly(edata);
 }
 
+#ifdef JEMALLOC_JET
 prof_recent_t *
 edata_prof_recent_alloc_get_no_lock_test(const edata_t *edata) {
 	cassert(config_prof);
 	return edata_prof_recent_alloc_get_no_lock(edata);
 }
+#endif
 
 static inline prof_recent_t *
 edata_prof_recent_alloc_get(tsd_t *tsd, const edata_t *edata) {

src/sc.c

@@ -13,7 +13,7 @@
  * at least the damage is compartmentalized to this file.
  */
 
-size_t
+JET_EXTERN size_t
 reg_size_compute(int lg_base, int lg_delta, int ndelta) {
 	return (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta);
 }

test/analyze/rand.c

@@ -1,5 +1,7 @@
 #include "test/jemalloc_test.h"
 
+extern uint64_t prof_sample_new_event_wait(tsd_t *tsd);
+
 /******************************************************************************/
 
 /*

test/include/test/hpa.h

@@ -0,0 +1,10 @@
+#ifndef TEST_HPA_H
+#define TEST_HPA_H
+
+/*
+ * Static in production builds (JET_EXTERN); exported only for the unit tests
+ * that share it (hpa_central, psset).
+ */
+extern bool hpa_hugepage_size_exceeds_limit(void);
+
+#endif /* TEST_HPA_H */

test/include/test/hpdata.h

@@ -0,0 +1,10 @@
+#ifndef TEST_HPDATA_H
+#define TEST_HPDATA_H
+
+/*
+ * Static in production builds (JET_EXTERN); exported only for the unit tests
+ * that share it (hpdata, psset).
+ */
+extern void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
+
+#endif /* TEST_HPDATA_H */

test/include/test/prof_data.h

@@ -0,0 +1,12 @@
+#ifndef TEST_PROF_DATA_H
+#define TEST_PROF_DATA_H
+
+/*
+ * Static in production builds (#ifdef JEMALLOC_JET); exported only for the unit
+ * tests that share them (prof_reset, prof_accum, prof_active, prof_tctx).
+ */
+extern size_t prof_tdata_count(void);
+extern size_t prof_bt_count(void);
+extern void   prof_cnt_all(prof_cnt_t *cnt_all);
+
+#endif /* TEST_PROF_DATA_H */

test/unit/base.c

@@ -2,6 +2,8 @@
 
 #include "test/extent_hooks.h"
 
+extern ehooks_t *base_ehooks_get_for_metadata(base_t *base);
+
 static extent_hooks_t hooks_null = {
     extent_alloc_hook, NULL, /* dalloc */
     NULL,                    /* destroy */

test/unit/bin.c

@@ -2,6 +2,15 @@
 
 extern void bin_dalloc_locked_begin(
     bin_dalloc_locked_info_t *info, szind_t binind);
+extern void *bin_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info);
+extern void bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab);
+extern void bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab);
+extern edata_t *bin_slabs_nonfull_tryget(bin_t *bin);
+extern void bin_slabs_full_insert(bool is_auto, bin_t *bin, edata_t *slab);
+extern void bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin);
+extern void bin_lower_slab(
+    tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin);
+extern void bin_dalloc_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin);
 
 #define INVALID_ARENA_IND ((1U << MALLOCX_ARENA_BITS) - 1)
 

test/unit/bitmap.c

@@ -2,6 +2,9 @@
 
 #include "test/nbits.h"
 
+extern void   bitmap_info_init(bitmap_info_t *binfo, size_t nbits);
+extern size_t bitmap_size(const bitmap_info_t *binfo);
+
 static void
 test_bitmap_initializer_body(const bitmap_info_t *binfo, size_t nbits) {
 	bitmap_info_t binfo_dyn;

test/unit/buf_writer.c

@@ -2,6 +2,8 @@
 
 #include "jemalloc/internal/buf_writer.h"
 
+extern void buf_writer_flush(buf_writer_t *buf_writer);
+
 #define TEST_BUF_SIZE 16
 #define UNIT_MAX (TEST_BUF_SIZE * 3)
 

test/unit/ckh.c

@@ -1,5 +1,10 @@
 #include "test/jemalloc_test.h"
 
+extern void ckh_string_hash(const void *key, size_t r_hash[2]);
+extern bool ckh_string_keycomp(const void *k1, const void *k2);
+extern void ckh_pointer_hash(const void *key, size_t r_hash[2]);
+extern bool ckh_pointer_keycomp(const void *k1, const void *k2);
+
 TEST_BEGIN(test_new_delete) {
 	tsd_t *tsd;
 	ckh_t  ckh;

test/unit/hpa_central.c

@@ -1,5 +1,7 @@
 #include "test/jemalloc_test.h"
 
+#include "test/hpa.h"
+
 #include "jemalloc/internal/hpa.h"
 #include "jemalloc/internal/hpa_central.h"
 

test/unit/hpdata.c

@@ -1,5 +1,7 @@
 #include "test/jemalloc_test.h"
 
+#include "test/hpdata.h"
+
 #define HPDATA_ADDR ((void *)(10 * HUGEPAGE))
 #define HPDATA_AGE 123
 

test/unit/nstime.c

@@ -1,5 +1,8 @@
 #include "test/jemalloc_test.h"
 
+extern uint64_t nstime_ms(const nstime_t *time);
+extern void     nstime_isubtract(nstime_t *time, uint64_t subtrahend);
+
 #define BILLION UINT64_C(1000000000)
 
 TEST_BEGIN(test_nstime_init) {

test/unit/prof_accum.c

@@ -1,5 +1,6 @@
 #include "test/jemalloc_test.h"
 
+#include "test/prof_data.h"
 #include "jemalloc/internal/prof_data.h"
 #include "jemalloc/internal/prof_sys.h"
 

test/unit/prof_active.c

@@ -1,5 +1,6 @@
 #include "test/jemalloc_test.h"
 
+#include "test/prof_data.h"
 #include "jemalloc/internal/prof_data.h"
 
 static void

test/unit/prof_log.c

@@ -1,6 +1,13 @@
 #include "test/jemalloc_test.h"
 #include "jemalloc/internal/prof_log.h"
 
+extern size_t prof_log_bt_count(void);
+extern size_t prof_log_alloc_count(void);
+extern size_t prof_log_thr_count(void);
+extern bool   prof_log_is_logging(void);
+extern bool   prof_log_rep_check(void);
+extern void   prof_log_dummy_set(bool new_value);
+
 #define N_PARAM 100
 #define N_THREADS 10
 

test/unit/prof_recent.c

@@ -2,6 +2,11 @@
 
 #include "jemalloc/internal/prof_recent.h"
 
+extern edata_t *prof_recent_alloc_edata_get_no_lock_test(
+    const prof_recent_t *node);
+extern prof_recent_t *edata_prof_recent_alloc_get_no_lock_test(
+    const edata_t *edata);
+
 /* As specified in the shell script */
 #define OPT_ALLOC_MAX 3
 

test/unit/prof_reset.c

@@ -1,5 +1,6 @@
 #include "test/jemalloc_test.h"
 
+#include "test/prof_data.h"
 #include "jemalloc/internal/prof_data.h"
 #include "jemalloc/internal/prof_sys.h"
 

test/unit/prof_tctx.c

@@ -1,5 +1,6 @@
 #include "test/jemalloc_test.h"
 
+#include "test/prof_data.h"
 #include "jemalloc/internal/prof_data.h"
 
 TEST_BEGIN(test_prof_realloc) {

test/unit/psset.c

@@ -1,5 +1,8 @@
 #include "test/jemalloc_test.h"
 
+#include "test/hpa.h"
+#include "test/hpdata.h"
+
 #include "jemalloc/internal/psset.h"
 
 #define PAGESLAB_ADDR ((void *)(1234 * HUGEPAGE))

test/unit/sz.c

@@ -1,5 +1,7 @@
 #include "test/jemalloc_test.h"
 
+extern size_t reg_size_compute(int lg_base, int lg_delta, int ndelta);
+
 TEST_BEGIN(test_sz_psz2ind) {
 	/*
 	 * Testing page size classes which reside prior to the regular group