haiwan
/
ck
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ck_bitmap: more bulk operations, less space, more tests

Browse Source 
* new bulk rmw: intersection, intersection_negate (with complement),
* new bulk reads: empty, full, count, count_intersect

* ck_bitmap_iterator fixes: avoid out of bounds read on empty bitmaps,
  and ck_bitmap_next is marginally faster on sparse bitmaps .

* less space: the bitmap itself is an array of unsigned int, which
  eliminates alignment padding between the n_bit field and the bitmap.

* more unit tests.
ck_pring
Paul Khuong 11 years ago
parent 96fdf1efbd
commit f06a5c4a71
2 changed files with 433 additions and 93 deletions
Show Stats Download Patch File Download Diff File

356
include/ck_bitmap.h
Unescape View File

@ -1,6 +1,7 @@
/*
* Copyright 2012-2014 Samy Al Bahra
* Copyright 2012-2014 AppNexus, Inc.
* Copyright 2014 Paul Khuong.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -36,37 +37,22 @@
#include <stddef.h>
#include <string.h>
#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_STORE_64) && \
defined(CK_F_PR_AND_64) && defined(CK_F_PR_OR_64)
#define CK_BITMAP_WORD uint64_t
#define CK_BITMAP_SHIFT 6
#define CK_BITMAP_STORE(x, y) ck_pr_store_64(x, y)
#define CK_BITMAP_LOAD(x) ck_pr_load_64(x)
#define CK_BITMAP_OR(x, y) ck_pr_or_64(x, y)
#define CK_BITMAP_AND(x, y) ck_pr_and_64(x, y)
#elif defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_STORE_32) && \
defined(CK_F_PR_AND_32) && defined(CK_F_PR_OR_32)
#define CK_BITMAP_WORD uint32_t
#define CK_BITMAP_SHIFT 5
#define CK_BITMAP_STORE(x, y) ck_pr_store_32(x, y)
#define CK_BITMAP_LOAD(x) ck_pr_load_32(x)
#define CK_BITMAP_OR(x, y) ck_pr_or_32(x, y)
#define CK_BITMAP_AND(x, y) ck_pr_and_32(x, y)
#else
#if !defined(CK_F_PR_LOAD_UINT) || !defined(CK_F_PR_STORE_UINT) || \
!defined(CK_F_PR_AND_UINT) || !defined(CK_F_PR_OR_UINT) || \
!defined(CK_F_CC_CTZ)
#error "ck_bitmap is not supported on your platform."
#endif /* These are all internal functions. */
#endif
#define CK_BITMAP_PTR(x, i) ((x) + ((i) >> CK_BITMAP_SHIFT))
#define CK_BITMAP_BLOCK (sizeof(CK_BITMAP_WORD) * CHAR_BIT)
#define CK_BITMAP_MASK (CK_BITMAP_BLOCK - 1)
#define CK_BITMAP_BLOCKS(n) \
(((n) + (CK_BITMAP_BLOCK - 1)) / CK_BITMAP_BLOCK)
#define CK_BITMAP_BLOCK (sizeof(unsigned int) * CHAR_BIT)
#define CK_BITMAP_BIT(i) (1U << ((i) % CK_BITMAP_BLOCK))
#define CK_BITMAP_PTR(x, i) ((x) + ((i) / CK_BITMAP_BLOCK))
#define CK_BITMAP_BLOCKS(n) (((n) + CK_BITMAP_BLOCK - 1) / CK_BITMAP_BLOCK)
#define CK_BITMAP_INSTANCE(n_entries) \
union { \
struct { \
unsigned int n_bits; \
CK_BITMAP_WORD map[CK_BITMAP_BLOCKS(n_entries)];\
unsigned int map[CK_BITMAP_BLOCKS(n_entries)]; \
} content; \
struct ck_bitmap bitmap; \
}
@ -83,17 +69,35 @@
#define CK_BITMAP_SET(a, b) \
ck_bitmap_set(&(a)->bitmap, (b))
#define CK_BITMAP_RESET(a, b) \
ck_bitmap_reset(&(a)->bitmap, (b))
#define CK_BITMAP_TEST(a, b) \
ck_bitmap_test(&(a)->bitmap, (b))
#define CK_BITMAP_UNION(a, b) \
ck_bitmap_union(&(a)->bitmap, &(b)->bitmap)
#define CK_BITMAP_RESET(a, b) \
ck_bitmap_reset(&(a)->bitmap, (b))
#define CK_BITMAP_INTERSECTION(a, b) \
ck_bitmap_intersection(&(a)->bitmap, &(b)->bitmap)
#define CK_BITMAP_INTERSECTION_NEGATE(a, b) \
ck_bitmap_intersection_negate(&(a)->bitmap, &(b)->bitmap)
#define CK_BITMAP_CLEAR(a) \
ck_bitmap_clear(&(a)->bitmap)
#define CK_BITMAP_TEST(a, b) \
ck_bitmap_test(&(a)->bitmap, (b))
#define CK_BITMAP_EMPTY(a, b) \
ck_bitmap_empty(&(a)->bitmap, b)
#define CK_BITMAP_FULL(a, b) \
ck_bitmap_full(&(a)->bitmap, b)
#define CK_BITMAP_COUNT(a, b) \
ck_bitmap_count(&(a)->bitmap, b)
#define CK_BITMAP_COUNT_INTERSECT(a, b, c) \
ck_bitmap_count_intersect(&(a)->bitmap, b, c)
#define CK_BITMAP_BITS(a) \
ck_bitmap_bits(&(a)->bitmap)
@ -106,14 +110,13 @@
struct ck_bitmap {
unsigned int n_bits;
CK_BITMAP_WORD map[];
unsigned int map[];
};
typedef struct ck_bitmap ck_bitmap_t;
struct ck_bitmap_iterator {
CK_BITMAP_WORD cache;
unsigned int cache;
unsigned int n_block;
unsigned int n_bit;
unsigned int n_limit;
};
typedef struct ck_bitmap_iterator ck_bitmap_iterator_t;
@ -122,7 +125,7 @@ CK_CC_INLINE static unsigned int
ck_bitmap_base(unsigned int n_bits)
{
return CK_BITMAP_BLOCKS(n_bits) * sizeof(CK_BITMAP_WORD);
return CK_BITMAP_BLOCKS(n_bits) * sizeof(unsigned int);
}
/*
@ -136,24 +139,68 @@ ck_bitmap_size(unsigned int n_bits)
return ck_bitmap_base(n_bits) + sizeof(struct ck_bitmap);
}
/*
* Returns total number of bits in specified bitmap.
*/
CK_CC_INLINE static unsigned int
ck_bitmap_bits(const struct ck_bitmap *bitmap)
{
return bitmap->n_bits;
}
/*
* Returns a pointer to the bit buffer associated
* with the specified bitmap.
*/
CK_CC_INLINE static void *
ck_bitmap_buffer(struct ck_bitmap *bitmap)
{
return bitmap->map;
}
/*
* Sets the bit at the offset specified in the second argument.
*/
CK_CC_INLINE static void
ck_bitmap_set(struct ck_bitmap *bitmap, unsigned int n)
{
CK_BITMAP_WORD mask = 0x1ULL << (n & CK_BITMAP_MASK);
CK_BITMAP_OR(CK_BITMAP_PTR(bitmap->map, n), mask);
ck_pr_or_uint(CK_BITMAP_PTR(bitmap->map, n), CK_BITMAP_BIT(n));
return;
}
/*
* Resets the bit at the offset specified in the second argument.
*/
CK_CC_INLINE static void
ck_bitmap_reset(struct ck_bitmap *bitmap, unsigned int n)
{
ck_pr_and_uint(CK_BITMAP_PTR(bitmap->map, n), ~CK_BITMAP_BIT(n));
return;
}
/*
* Determines whether the bit at offset specified in the
* second argument is set.
*/
CK_CC_INLINE static bool
ck_bitmap_test(const struct ck_bitmap *bitmap, unsigned int n)
{
unsigned int block;
block = ck_pr_load_uint(CK_BITMAP_PTR(bitmap->map, n));
return block & CK_BITMAP_BIT(n);
}
/*
* Combines bits from second bitmap into the first bitmap. This is not a
* linearized operation with respect to the complete bitmap.
*/
CK_CC_INLINE static void
ck_bitmap_union(struct ck_bitmap *dst, struct ck_bitmap *src)
ck_bitmap_union(struct ck_bitmap *dst, const struct ck_bitmap *src)
{
unsigned int n;
unsigned int n_buckets = dst->n_bits;
@ -162,21 +209,62 @@ ck_bitmap_union(struct ck_bitmap *dst, struct ck_bitmap *src)
n_buckets = src->n_bits;
n_buckets = CK_BITMAP_BLOCKS(n_buckets);
for (n = 0; n < n_buckets; n++)
CK_BITMAP_OR(&dst->map[n], src->map[n]);
for (n = 0; n < n_buckets; n++) {
ck_pr_or_uint(&dst->map[n],
ck_pr_load_uint(&src->map[n]));
}
return;
}
/*
* Resets the bit at the offset specified in the second argument.
* Intersects bits from second bitmap into the first bitmap. This is
* not a linearized operation with respect to the complete bitmap.
* Any trailing bit in dst is cleared.
*/
CK_CC_INLINE static void
ck_bitmap_reset(struct ck_bitmap *bitmap, unsigned int n)
ck_bitmap_intersection(struct ck_bitmap *dst, const struct ck_bitmap *src)
{
unsigned int n;
unsigned int n_buckets = dst->n_bits;
unsigned int n_intersect = n_buckets;
if (src->n_bits < n_intersect)
n_intersect = src->n_bits;
n_buckets = CK_BITMAP_BLOCKS(n_buckets);
n_intersect = CK_BITMAP_BLOCKS(n_intersect);
for (n = 0; n < n_intersect; n++) {
ck_pr_and_uint(&dst->map[n],
ck_pr_load_uint(&src->map[n]));
}
for (; n < n_buckets; n++)
ck_pr_store_uint(&dst->map[n], 0);
return;
}
/*
* Intersects the complement of bits from second bitmap into the first
* bitmap. This is not a linearized operation with respect to the
* complete bitmap. Any trailing bit in dst is left as is.
*/
CK_CC_INLINE static void
ck_bitmap_intersection_negate(struct ck_bitmap *dst, const struct ck_bitmap *src)
{
CK_BITMAP_WORD mask = ~(0x1ULL << (n & CK_BITMAP_MASK));
unsigned int n;
unsigned int n_intersect = dst->n_bits;
if (src->n_bits < n_intersect)
n_intersect = src->n_bits;
n_intersect = CK_BITMAP_BLOCKS(n_intersect);
for (n = 0; n < n_intersect; n++) {
ck_pr_and_uint(&dst->map[n],
(~ck_pr_load_uint(&src->map[n])));
}
CK_BITMAP_AND(CK_BITMAP_PTR(bitmap->map, n), mask);
return;
}
@ -187,48 +275,139 @@ ck_bitmap_reset(struct ck_bitmap *bitmap, unsigned int n)
CK_CC_INLINE static void
ck_bitmap_clear(struct ck_bitmap *bitmap)
{
unsigned int n_buckets = ck_bitmap_base(bitmap->n_bits) / sizeof(CK_BITMAP_WORD);
unsigned int n_buckets = ck_bitmap_base(bitmap->n_bits) / sizeof(unsigned int);
unsigned int i;
for (i = 0; i < n_buckets; i++)
CK_BITMAP_STORE(&bitmap->map[i], 0);
ck_pr_store_uint(&bitmap->map[i], 0);
return;
}
/*
* Determines whether the bit at offset specified in the
* second argument is set.
* Returns true if the first limit bits in bitmap are cleared. If
* limit is greater than the bitmap size, limit is truncated to that
* size.
*/
CK_CC_INLINE static bool
ck_bitmap_test(struct ck_bitmap *bitmap, unsigned int n)
ck_bitmap_empty(const ck_bitmap_t *bitmap, unsigned int limit)
{
CK_BITMAP_WORD mask = 0x1ULL << (n & CK_BITMAP_MASK);
CK_BITMAP_WORD block;
unsigned int i, words, slop;
if (limit > bitmap->n_bits)
limit = bitmap->n_bits;
words = limit / CK_BITMAP_BLOCK;
slop = limit % CK_BITMAP_BLOCK;
for (i = 0; i < words; i++) {
if (ck_pr_load_uint(&bitmap->map[i]) != 0) {
return false;
}
}
if (slop > 0) {
unsigned int word;
word = ck_pr_load_uint(&bitmap->map[i]);
if ((word & ((1U << slop) - 1)) != 0)
return false;
}
block = CK_BITMAP_LOAD(CK_BITMAP_PTR(bitmap->map, n));
return block & mask;
return true;
}
/*
* Returns total number of bits in specified bitmap.
* Returns true if the first limit bits in bitmap are set. If limit
* is greater than the bitmap size, limit is truncated to that size.
*/
CK_CC_UNUSED static bool
ck_bitmap_full(const ck_bitmap_t *bitmap, unsigned int limit)
{
unsigned int i, slop, words;
if (limit > bitmap->n_bits) {
limit = bitmap->n_bits;
}
words = limit / CK_BITMAP_BLOCK;
slop = limit % CK_BITMAP_BLOCK;
for (i = 0; i < words; i++) {
if (ck_pr_load_uint(&bitmap->map[i]) != -1U)
return false;
}
if (slop > 0) {
unsigned int word;
word = ~ck_pr_load_uint(&bitmap->map[i]);
if ((word & ((1U << slop) - 1)) != 0)
return false;
}
return true;
}
/*
* Returns the number of set bit in bitmap, upto (and excluding)
* limit. If limit is greater than the bitmap size, it is truncated
* to that size.
*/
CK_CC_INLINE static unsigned int
ck_bitmap_bits(struct ck_bitmap *bitmap)
ck_bitmap_count(const ck_bitmap_t *bitmap, unsigned int limit)
{
unsigned int count, i, slop, words;
return bitmap->n_bits;
if (limit > bitmap->n_bits)
limit = bitmap->n_bits;
words = limit / CK_BITMAP_BLOCK;
slop = limit % CK_BITMAP_BLOCK;
for (i = 0, count = 0; i < words; i++)
count += ck_cc_popcount(ck_pr_load_uint(&bitmap->map[i]));
if (slop > 0) {
unsigned int word;
word = ck_pr_load_uint(&bitmap->map[i]);
count += ck_cc_popcount(word & ((1U << slop) - 1));
}
return count;
}
/*
* Returns a pointer to the bit buffer associated
* with the specified bitmap.
* Returns the number of set bit in the intersection of two bitmaps,
* upto (and exclusing) limit. If limit is greater than either bitmap
* size, it is truncated to the smallest.
*/
CK_CC_INLINE static void *
ck_bitmap_buffer(struct ck_bitmap *bitmap)
CK_CC_INLINE static unsigned int
ck_bitmap_count_intersect(const ck_bitmap_t *x, const ck_bitmap_t *y, unsigned int limit)
{
unsigned int count, i, slop, words;
return bitmap->map;
if (limit > x->n_bits)
limit = x->n_bits;
if (limit > y->n_bits)
limit = y->n_bits;
words = limit / CK_BITMAP_BLOCK;
slop = limit % CK_BITMAP_BLOCK;
for (i = 0, count = 0; i < words; i++) {
unsigned int xi, yi;
xi = ck_pr_load_uint(&x->map[i]);
yi = ck_pr_load_uint(&y->map[i]);
count += ck_cc_popcount(xi & yi);
}
if (slop > 0) {
unsigned int word, xi, yi;
xi = ck_pr_load_uint(&x->map[i]);
yi = ck_pr_load_uint(&y->map[i]);
word = xi & yi;
count += ck_cc_popcount(word & ((1U << slop) - 1));
}
return count;
}
/*
@ -247,34 +426,31 @@ ck_bitmap_init(struct ck_bitmap *bitmap,
memset(bitmap->map, -(int)set, base);
if (set == true) {
CK_BITMAP_WORD b;
if (n_bits < CK_BITMAP_BLOCK)
b = n_bits;
else
b = n_bits % CK_BITMAP_BLOCK;
unsigned int b = n_bits % CK_BITMAP_BLOCK;
if (b == 0)
return;
bitmap->map[base / sizeof(CK_BITMAP_WORD) - 1] &= (1ULL << b) - 1ULL;
*CK_BITMAP_PTR(bitmap->map, n_bits - 1) &= (1U << b) - 1U;
}
return;
}
/*
* Initialize iterator for use with provided bitmap.
*/
CK_CC_INLINE static void
ck_bitmap_iterator_init(struct ck_bitmap_iterator *i, struct ck_bitmap *bitmap)
ck_bitmap_iterator_init(struct ck_bitmap_iterator *i, const struct ck_bitmap *bitmap)
{
i->n_block = 0;
i->n_bit = 0;
i->n_limit = CK_BITMAP_BLOCKS(bitmap->n_bits);
i->cache = CK_BITMAP_LOAD(&bitmap->map[i->n_block]);
if (i->n_limit > 0) {
i->cache = ck_pr_load_uint(&bitmap->map[0]);
} else {
i->cache = 0;
}
return;
}
@ -282,38 +458,34 @@ ck_bitmap_iterator_init(struct ck_bitmap_iterator *i, struct ck_bitmap *bitmap)
* Iterate to next bit.
*/
CK_CC_INLINE static bool
ck_bitmap_next(struct ck_bitmap *bitmap,
ck_bitmap_next(const struct ck_bitmap *bitmap,
struct ck_bitmap_iterator *i,
unsigned int *bit)
{
unsigned int cache = i->cache;
unsigned int n_block = i->n_block;
unsigned int n_limit = i->n_limit;
/* Load next bitmap block. */
for (;;) {
while (i->n_bit < CK_BITMAP_BLOCK) {
unsigned int previous = i->n_bit++;
if (i->cache & 1) {
*bit = previous + (CK_BITMAP_BLOCK * i->n_block);
i->cache >>= 1;
return true;
}
if (cache == 0) {
if (n_block >= n_limit)
return false;
i->cache >>= 1;
if (i->cache == 0)
break;
for (n_block++; n_block < n_limit; n_block++) {
cache = ck_pr_load_uint(&bitmap->map[n_block]);
if (cache != 0)
goto non_zero;
}
i->n_bit = 0;
i->n_block++;
if (i->n_block >= i->n_limit)
return false;
i->cache = CK_BITMAP_LOAD(&bitmap->map[i->n_block]);
i->cache = 0;
i->n_block = n_block;
return false;
}
return false;
non_zero:
*bit = CK_BITMAP_BLOCK * n_block + ck_cc_ctz(cache);
i->cache = cache & (cache - 1);
i->n_block = n_block;
return true;
}
#endif /* _CK_BITMAP_H */

170
regressions/ck_bitmap/validate/serial.c
Unescape View File

@ -2,6 +2,7 @@
* Copyright 2012-2014 Samy Al Bahra.
* Copyright 2012-2014 AppNexus, Inc.
* Copyright 2012 Shreyas Prasad.
* Copyright 2014 Paul Khuong.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -127,10 +128,171 @@ test(ck_bitmap_t *bits, unsigned int n_length, bool initial)
return;
}
static void
test_init(bool init)
{
ck_bitmap_t *bitmap;
size_t bytes;
unsigned int i;
bytes = ck_bitmap_size(length);
bitmap = malloc(bytes);
memset(bitmap, random(), bytes);
ck_bitmap_init(bitmap, length, init);
if (ck_bitmap_bits(bitmap) != length) {
ck_error("ERROR: Expected length %u got %u\n",
length, ck_bitmap_bits(bitmap));
}
for (i = 0; i < length; i++) {
if (ck_bitmap_test(bitmap, i) != init) {
ck_error("ERROR: Expected bit %i at index %u, got %i\n",
(int)init, i, (int)(!init));
}
}
free(bitmap);
}
static ck_bitmap_t *
random_init()
{
ck_bitmap_t *bitmap;
unsigned int i;
bitmap = malloc(ck_bitmap_size(length));
ck_bitmap_init(bitmap, length, false);
for (i = 0; i < length; i++) {
if (random() & 1) {
ck_bitmap_set(bitmap, i);
}
}
return bitmap;
}
static ck_bitmap_t *
copy(const ck_bitmap_t *src)
{
ck_bitmap_t *bitmap;
size_t bytes = ck_bitmap_size(ck_bitmap_bits(src));
bitmap = malloc(bytes);
memcpy(bitmap, src, bytes);
return bitmap;
}
static void
test_counts(const ck_bitmap_t *x, const ck_bitmap_t *y)
{
unsigned int count = 0;
unsigned int count_intersect = 0;
unsigned int i;
for (i = 0; i <= length * 2; i++) {
unsigned actual_limit = i;
unsigned int r;
bool check;
if (actual_limit > ck_bitmap_bits(x))
actual_limit = ck_bitmap_bits(x);
check = ck_bitmap_empty(x, i);
if (check != (count == 0)) {
ck_error("ck_bitmap_empty(%u): got %i expected %i\n",
i, (int)check, (int)(count == 0));
}
check = ck_bitmap_full(x, i);
if (check != (count == actual_limit)) {
ck_error("ck_bitmap_full(%u): got %i expected %i\n",
i, (int)check, (int)(count == i));
}
r = ck_bitmap_count(x, i);
if (r != count) {
ck_error("ck_bitmap_count(%u): got %u expected %u\n",
i, r, count);
}
r = ck_bitmap_count_intersect(x, y, i);
if (r != count_intersect) {
ck_error("ck_bitmap_count_intersect(%u): got %u expected %u\n",
i, r, count_intersect);
}
if (i < length) {
count += ck_bitmap_test(x, i);
count_intersect += ck_bitmap_test(x, i) & ck_bitmap_test(y, i);
}
}
}
static void
random_test(unsigned int seed)
{
ck_bitmap_t *x, *x_copy, *y;
unsigned int i;
srandom(seed);
test_init(false);
test_init(true);
x = random_init();
y = random_init();
#define TEST(routine, expected) do { \
x_copy = copy(x); \
routine(x_copy, y); \
for (i = 0; i < length; i++) { \
bool xi = ck_bitmap_test(x, i); \
bool yi = ck_bitmap_test(y, i); \
bool ri = ck_bitmap_test(x_copy, i); \
bool wanted = expected(xi, yi); \
\
if (ri != wanted) { \
ck_error("In " #routine " at %u: " \
"got %i expected %i\n", \
i, ri, wanted); \
} \
} \
free(x_copy); \
} while (0)
#define OR(x, y) (x | y)
#define AND(x, y) (x & y)
#define ANDC2(x, y) (x & (~y))
TEST(ck_bitmap_union, OR);
TEST(ck_bitmap_intersection, AND);
TEST(ck_bitmap_intersection_negate, ANDC2);
#undef ANDC2
#undef AND
#undef OR
#undef TEST
test_counts(x, y);
for (i = 0; i < 4; i++) {
ck_bitmap_init(x, length, i & 1);
ck_bitmap_init(y, length, i >> 1);
test_counts(x, y);
}
free(y);
free(x);
}
int
main(int argc, char *argv[])
{
unsigned int bytes, base;
size_t i, j;
if (argc >= 2) {
length = atoi(argv[1]);
@ -177,6 +339,12 @@ main(int argc, char *argv[])
ck_error("ERROR: Expected bit to be reset.\n");
}
for (i = 0; i < 4 * sizeof(unsigned int) * CHAR_BIT; i++) {
length = i;
for (j = 0; j < 10; j++) {
random_test(i * 10 + j);
}
}
return 0;
}

Write Preview
Loading…
Cancel
Save