jemalloc/include/jemalloc/internal/tsd_generic.h

#ifdef JEMALLOC_INTERNAL_TSD_GENERIC_H
#	error This file should be included only once, by tsd.h.
#endif
#define JEMALLOC_INTERNAL_TSD_GENERIC_H

#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/tsd_internals.h"
#include "jemalloc/internal/tsd_types.h"

typedef struct tsd_init_block_s tsd_init_block_t;
struct tsd_init_block_s {
	ql_elm(tsd_init_block_t) link;
	pthread_t thread;
	void     *data;
};

/* Defined in tsd.c, to allow the mutex headers to have tsd dependencies. */
typedef struct tsd_init_head_s tsd_init_head_t;

typedef struct {
	bool  initialized;
	tsd_t val;
} tsd_wrapper_t;

void *tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block);
void  tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);

extern pthread_key_t   tsd_tsd;
extern pthread_key_t   tsd_thread_initialized_tsd;
extern tsd_init_head_t tsd_init_head;
extern tsd_wrapper_t   tsd_boot_wrapper;
extern bool            tsd_booted;

/* Initialization/cleanup. */
JEMALLOC_ALWAYS_INLINE void
tsd_cleanup_wrapper(void *arg) {
	tsd_wrapper_t *wrapper = (tsd_wrapper_t *)arg;

	if (wrapper->initialized) {
		wrapper->initialized = false;
		tsd_cleanup(&wrapper->val);
		if (wrapper->initialized) {
			/* Trigger another cleanup round. */
			if (pthread_setspecific(tsd_tsd, (void *)wrapper)
			    != 0) {
				malloc_write("<jemalloc>: Error setting TSD\n");
				if (opt_abort) {
					abort();
				}
			}
			return;
		}
	}
	/*
	 * Leave tsd_thread_initialized_tsd set.  It has no destructor and
	 * intentionally outlives the TSD wrapper, so later jemalloc calls can
	 * distinguish this fully-torn-down thread from one that has never
	 * initialized TSD.
	 */
	malloc_tsd_dalloc(wrapper);
}

JEMALLOC_ALWAYS_INLINE void
tsd_wrapper_set(tsd_wrapper_t *wrapper) {
	if (unlikely(!tsd_booted)) {
		return;
	}
	if (pthread_setspecific(tsd_tsd, (void *)wrapper) != 0) {
		malloc_write("<jemalloc>: Error setting TSD\n");
		abort();
	}
	if (pthread_setspecific(tsd_thread_initialized_tsd,
	    (void *)&tsd_thread_initialized_tsd) != 0) {
		malloc_write("<jemalloc>: Error setting TSD\n");
		abort();
	}
}

JEMALLOC_ALWAYS_INLINE tsd_wrapper_t *
tsd_wrapper_get(bool init) {
	tsd_wrapper_t *wrapper;

	if (unlikely(!tsd_booted)) {
		return &tsd_boot_wrapper;
	}

	wrapper = (tsd_wrapper_t *)pthread_getspecific(tsd_tsd);

	if (init && unlikely(wrapper == NULL)) {
		tsd_init_block_t block;
		wrapper = (tsd_wrapper_t *)tsd_init_check_recursion(
		    &tsd_init_head, &block);
		if (wrapper) {
			return wrapper;
		}
		wrapper = (tsd_wrapper_t *)malloc_tsd_malloc(
		    sizeof(tsd_wrapper_t));
		block.data = (void *)wrapper;
		if (wrapper == NULL) {
			malloc_write("<jemalloc>: Error allocating TSD\n");
			abort();
		} else {
			wrapper->initialized = false;
			JEMALLOC_DIAGNOSTIC_PUSH
			JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
			tsd_t initializer = TSD_INITIALIZER;
			JEMALLOC_DIAGNOSTIC_POP
			wrapper->val = initializer;
		}
		tsd_wrapper_set(wrapper);
		tsd_init_finish(&tsd_init_head, &block);
	}
	return wrapper;
}

JEMALLOC_ALWAYS_INLINE bool
tsd_boot0(void) {
	tsd_wrapper_t   *wrapper;
	tsd_init_block_t block;

	wrapper = (tsd_wrapper_t *)tsd_init_check_recursion(
	    &tsd_init_head, &block);
	if (wrapper) {
		return false;
	}
	block.data = &tsd_boot_wrapper;
	if (pthread_key_create(&tsd_tsd, tsd_cleanup_wrapper) != 0) {
		return true;
	}
	/*
	 * This key has no destructor.  It records that the current thread once
	 * had jemalloc TSD, so a later NULL tsd_tsd value means teardown has
	 * finished rather than initialization not having happened yet.
	 */
	if (pthread_key_create(&tsd_thread_initialized_tsd, NULL) != 0) {
		pthread_key_delete(tsd_tsd);
		return true;
	}
	tsd_booted = true;
	tsd_wrapper_set(&tsd_boot_wrapper);
	tsd_init_finish(&tsd_init_head, &block);
	return false;
}

JEMALLOC_ALWAYS_INLINE void
tsd_boot1(void) {
	tsd_wrapper_t *wrapper;
	wrapper = (tsd_wrapper_t *)malloc_tsd_malloc(sizeof(tsd_wrapper_t));
	if (wrapper == NULL) {
		malloc_write("<jemalloc>: Error allocating TSD\n");
		abort();
	}
	tsd_boot_wrapper.initialized = false;
	tsd_cleanup(&tsd_boot_wrapper.val);
	wrapper->initialized = false;
	JEMALLOC_DIAGNOSTIC_PUSH
	JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
	tsd_t initializer = TSD_INITIALIZER;
	JEMALLOC_DIAGNOSTIC_POP
	wrapper->val = initializer;
	tsd_wrapper_set(wrapper);
}

JEMALLOC_ALWAYS_INLINE bool
tsd_boot(void) {
	if (tsd_boot0()) {
		return true;
	}
	tsd_boot1();
	return false;
}

JEMALLOC_ALWAYS_INLINE bool
tsd_booted_get(void) {
	return tsd_booted;
}

JEMALLOC_ALWAYS_INLINE bool
tsd_get_allocates(void) {
	return true;
}

JEMALLOC_ALWAYS_INLINE bool
tsd_teardown_done(void) {
	return tsd_booted && pthread_getspecific(tsd_tsd) == NULL
	    && pthread_getspecific(tsd_thread_initialized_tsd) != NULL;
}

/* Get/set. */
JEMALLOC_ALWAYS_INLINE tsd_t *
tsd_get(bool init) {
	tsd_wrapper_t *wrapper;

	assert(tsd_booted);
	wrapper = tsd_wrapper_get(init);
	if (tsd_get_allocates() && !init && wrapper == NULL) {
		return NULL;
	}
	return &wrapper->val;
}

JEMALLOC_ALWAYS_INLINE void
tsd_set(tsd_t *val) {
	tsd_wrapper_t *wrapper;

	assert(tsd_booted);
	wrapper = tsd_wrapper_get(true);
	if (likely(&wrapper->val != val)) {
		wrapper->val = *(val);
	}
	wrapper->initialized = true;
}