mirror of
https://github.com/jemalloc/jemalloc.git
synced 2026-07-22 04:03:11 +00:00
b8823ab026
For small bitmaps, a linear scan of the bitmap is slightly faster than a tree search - bitmap_t is more compact, and there are fewer writes since we don't have to propogate state transitions up the tree. On x86_64 with the current settings, I'm seeing ~.5%-1% CPU improvement in production canaries with this change. The old tree code is left since 32bit sizes are much larger (and ffsl smaller), and maybe the run sizes will change in the future. This resolves #339.
115 lines
2.8 KiB
C
115 lines
2.8 KiB
C
#define JEMALLOC_BITMAP_C_
|
|
#include "jemalloc/internal/jemalloc_internal.h"
|
|
|
|
/******************************************************************************/
|
|
|
|
#ifdef USE_TREE
|
|
|
|
void
|
|
bitmap_info_init(bitmap_info_t *binfo, size_t nbits)
|
|
{
|
|
unsigned i;
|
|
size_t group_count;
|
|
|
|
assert(nbits > 0);
|
|
assert(nbits <= (ZU(1) << LG_BITMAP_MAXBITS));
|
|
|
|
/*
|
|
* Compute the number of groups necessary to store nbits bits, and
|
|
* progressively work upward through the levels until reaching a level
|
|
* that requires only one group.
|
|
*/
|
|
binfo->levels[0].group_offset = 0;
|
|
group_count = BITMAP_BITS2GROUPS(nbits);
|
|
for (i = 1; group_count > 1; i++) {
|
|
assert(i < BITMAP_MAX_LEVELS);
|
|
binfo->levels[i].group_offset = binfo->levels[i-1].group_offset
|
|
+ group_count;
|
|
group_count = BITMAP_BITS2GROUPS(group_count);
|
|
}
|
|
binfo->levels[i].group_offset = binfo->levels[i-1].group_offset
|
|
+ group_count;
|
|
assert(binfo->levels[i].group_offset <= BITMAP_GROUPS_MAX);
|
|
binfo->nlevels = i;
|
|
binfo->nbits = nbits;
|
|
}
|
|
|
|
static size_t
|
|
bitmap_info_ngroups(const bitmap_info_t *binfo)
|
|
{
|
|
|
|
return (binfo->levels[binfo->nlevels].group_offset);
|
|
}
|
|
|
|
void
|
|
bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo)
|
|
{
|
|
size_t extra;
|
|
unsigned i;
|
|
|
|
/*
|
|
* Bits are actually inverted with regard to the external bitmap
|
|
* interface, so the bitmap starts out with all 1 bits, except for
|
|
* trailing unused bits (if any). Note that each group uses bit 0 to
|
|
* correspond to the first logical bit in the group, so extra bits
|
|
* are the most significant bits of the last group.
|
|
*/
|
|
memset(bitmap, 0xffU, bitmap_size(binfo));
|
|
extra = (BITMAP_GROUP_NBITS - (binfo->nbits & BITMAP_GROUP_NBITS_MASK))
|
|
& BITMAP_GROUP_NBITS_MASK;
|
|
if (extra != 0)
|
|
bitmap[binfo->levels[1].group_offset - 1] >>= extra;
|
|
for (i = 1; i < binfo->nlevels; i++) {
|
|
size_t group_count = binfo->levels[i].group_offset -
|
|
binfo->levels[i-1].group_offset;
|
|
extra = (BITMAP_GROUP_NBITS - (group_count &
|
|
BITMAP_GROUP_NBITS_MASK)) & BITMAP_GROUP_NBITS_MASK;
|
|
if (extra != 0)
|
|
bitmap[binfo->levels[i+1].group_offset - 1] >>= extra;
|
|
}
|
|
}
|
|
|
|
#else /* USE_TREE */
|
|
|
|
void
|
|
bitmap_info_init(bitmap_info_t *binfo, size_t nbits)
|
|
{
|
|
size_t i;
|
|
|
|
assert(nbits > 0);
|
|
assert(nbits <= (ZU(1) << LG_BITMAP_MAXBITS));
|
|
|
|
i = nbits >> LG_BITMAP_GROUP_NBITS;
|
|
if (nbits % BITMAP_GROUP_NBITS != 0)
|
|
i++;
|
|
binfo->ngroups = i;
|
|
binfo->nbits = nbits;
|
|
}
|
|
|
|
static size_t
|
|
bitmap_info_ngroups(const bitmap_info_t *binfo)
|
|
{
|
|
|
|
return (binfo->ngroups);
|
|
}
|
|
|
|
void
|
|
bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo)
|
|
{
|
|
size_t extra;
|
|
|
|
memset(bitmap, 0xffU, bitmap_size(binfo));
|
|
extra = (binfo->nbits % (binfo->ngroups * BITMAP_GROUP_NBITS));
|
|
if (extra != 0)
|
|
bitmap[binfo->ngroups - 1] >>= (BITMAP_GROUP_NBITS - extra);
|
|
}
|
|
|
|
#endif /* USE_TREE */
|
|
|
|
size_t
|
|
bitmap_size(const bitmap_info_t *binfo)
|
|
{
|
|
|
|
return (bitmap_info_ngroups(binfo) << LG_SIZEOF_BITMAP);
|
|
}
|