diff --git a/.gitignore b/.gitignore index 58ab781..a37e698 100644 --- a/.gitignore +++ b/.gitignore @@ -20,23 +20,27 @@ GRTAGS GTAGS ID regressions/ck_array/validate/serial -regressions/ck_cohort/benchmark/ck_cohort.LATENCY -regressions/ck_cohort/benchmark/ck_cohort.THROUGHPUT -regressions/ck_pflock/benchmark/latency -regressions/ck_pflock/benchmark/throughput -regressions/ck_pflock/validate/validate +regressions/ck_backoff/validate/validate +regressions/ck_bag/validate/order regressions/ck_barrier/benchmark/throughput regressions/ck_barrier/validate/barrier_centralized regressions/ck_barrier/validate/barrier_combining -regressions/ck_brlock/benchmark/latency -regressions/ck_brlock/validate/validate -regressions/ck_rwlock/benchmark/latency -regressions/ck_rwlock/validate/validate regressions/ck_barrier/validate/barrier_dissemination regressions/ck_barrier/validate/barrier_mcs regressions/ck_barrier/validate/barrier_tournament +regressions/ck_bitmap/validate/serial +regressions/ck_brlock/benchmark/latency +regressions/ck_brlock/benchmark/throughput +regressions/ck_brlock/validate/validate regressions/ck_bytelock/benchmark/latency regressions/ck_bytelock/validate/validate +regressions/ck_cohort/benchmark/ck_cohort.LATENCY +regressions/ck_cohort/benchmark/ck_cohort.LATENCY +regressions/ck_cohort/benchmark/ck_cohort.THROUGHPUT +regressions/ck_cohort/benchmark/ck_cohort.THROUGHPUT +regressions/ck_cohort/validate/validate +regressions/ck_epoch/validate/ck_epoch_poll +regressions/ck_epoch/validate/ck_epoch_synchronize regressions/ck_epoch/validate/ck_stack regressions/ck_epoch/validate/ck_stack_read regressions/ck_fifo/benchmark/latency @@ -47,9 +51,24 @@ regressions/ck_fifo/validate/ck_fifo_spsc_iterator regressions/ck_hp/benchmark/fifo_latency regressions/ck_hp/benchmark/stack_latency regressions/ck_hp/validate/ck_hp_fifo +regressions/ck_hp/validate/ck_hp_fifo_donner regressions/ck_hp/validate/ck_hp_stack regressions/ck_hp/validate/nbds_haz_test regressions/ck_hp/validate/serial +regressions/ck_hs/benchmark/parallel_bytestring +regressions/ck_hs/benchmark/parallel_bytestring.delete +regressions/ck_hs/benchmark/serial +regressions/ck_hs/validate/serial +regressions/ck_ht/benchmark/parallel_bytestring +regressions/ck_ht/benchmark/parallel_bytestring.delete +regressions/ck_ht/benchmark/parallel_direct +regressions/ck_ht/benchmark/serial +regressions/ck_ht/benchmark/serial.delete +regressions/ck_ht/validate/serial +regressions/ck_ht/validate/serial.delete +regressions/ck_pflock/benchmark/latency +regressions/ck_pflock/benchmark/throughput +regressions/ck_pflock/validate/validate regressions/ck_pr/benchmark/ck_pr_cas_64 regressions/ck_pr/benchmark/ck_pr_cas_64_2 regressions/ck_pr/benchmark/ck_pr_fas_64 @@ -73,10 +92,31 @@ regressions/ck_pr/validate/ck_pr_store regressions/ck_pr/validate/ck_pr_sub regressions/ck_pr/validate/ck_pr_unary regressions/ck_pr/validate/ck_pr_xor +regressions/ck_queue/validate/ck_list +regressions/ck_queue/validate/ck_slist +regressions/ck_queue/validate/ck_stailq +regressions/ck_rhs/benchmark/parallel_bytestring +regressions/ck_rhs/benchmark/serial +regressions/ck_rhs/validate/serial +regressions/ck_ring/benchmark/latency +regressions/ck_ring/validate/ck_ring_spmc +regressions/ck_ring/validate/ck_ring_spmc_template regressions/ck_ring/validate/ck_ring_spsc regressions/ck_ring/validate/ck_ring_spsc_template -regressions/ck_sequence/validate/ck_sequence +regressions/ck_rwcohort/benchmark/ck_neutral.LATENCY +regressions/ck_rwcohort/benchmark/ck_neutral.THROUGHPUT +regressions/ck_rwcohort/benchmark/ck_rp.LATENCY +regressions/ck_rwcohort/benchmark/ck_rp.THROUGHPUT +regressions/ck_rwcohort/benchmark/ck_wp.LATENCY +regressions/ck_rwcohort/benchmark/ck_wp.THROUGHPUT +regressions/ck_rwcohort/validate/ck_neutral +regressions/ck_rwcohort/validate/ck_rp +regressions/ck_rwcohort/validate/ck_wp +regressions/ck_rwlock/benchmark/latency +regressions/ck_rwlock/benchmark/throughput +regressions/ck_rwlock/validate/validate regressions/ck_sequence/benchmark/ck_sequence +regressions/ck_sequence/validate/ck_sequence regressions/ck_spinlock/benchmark/ck_anderson.LATENCY regressions/ck_spinlock/benchmark/ck_anderson.THROUGHPUT regressions/ck_spinlock/benchmark/ck_cas.LATENCY @@ -87,44 +127,26 @@ regressions/ck_spinlock/benchmark/ck_dec.LATENCY regressions/ck_spinlock/benchmark/ck_dec.THROUGHPUT regressions/ck_spinlock/benchmark/ck_fas.LATENCY regressions/ck_spinlock/benchmark/ck_fas.THROUGHPUT +regressions/ck_spinlock/benchmark/ck_hclh.LATENCY +regressions/ck_spinlock/benchmark/ck_hclh.THROUGHPUT regressions/ck_spinlock/benchmark/ck_mcs.LATENCY regressions/ck_spinlock/benchmark/ck_mcs.THROUGHPUT +regressions/ck_spinlock/benchmark/ck_spinlock.LATENCY +regressions/ck_spinlock/benchmark/ck_spinlock.THROUGHPUT regressions/ck_spinlock/benchmark/ck_ticket.LATENCY regressions/ck_spinlock/benchmark/ck_ticket.THROUGHPUT regressions/ck_spinlock/benchmark/ck_ticket_pb.LATENCY regressions/ck_spinlock/benchmark/ck_ticket_pb.THROUGHPUT regressions/ck_spinlock/benchmark/linux_spinlock.LATENCY regressions/ck_spinlock/benchmark/linux_spinlock.THROUGHPUT -regressions/ck_backoff/validate/validate -regressions/ck_bag/validate/order -regressions/ck_bitmap/validate/serial -regressions/ck_epoch/validate/ck_epoch_poll -regressions/ck_epoch/validate/ck_epoch_synchronize -regressions/ck_hp/validate/ck_hp_fifo_donner -regressions/ck_hs/benchmark/parallel_bytestring -regressions/ck_hs/benchmark/serial -regressions/ck_hs/validate/serial -regressions/ck_ht/benchmark/parallel_bytestring -regressions/ck_ht/benchmark/parallel_direct -regressions/ck_ht/benchmark/serial -regressions/ck_ht/validate/serial -regressions/ck_ring/benchmark/latency -regressions/ck_ring/validate/ck_ring_spmc -regressions/ck_ring/validate/ck_ring_spmc_template -regressions/ck_spinlock/benchmark/ck_spinlock.LATENCY -regressions/ck_spinlock/benchmark/ck_spinlock.THROUGHPUT -regressions/ck_spinlock/validate/ck_spinlock -regressions/ck_stack/validate/mpmc_trypair -regressions/ck_stack/validate/mpmc_trypop -regressions/ck_stack/validate/mpmc_trypush -regressions/ck_stack/validate/upmc_trypop -regressions/ck_stack/validate/upmc_trypush regressions/ck_spinlock/validate/ck_anderson regressions/ck_spinlock/validate/ck_cas regressions/ck_spinlock/validate/ck_clh regressions/ck_spinlock/validate/ck_dec regressions/ck_spinlock/validate/ck_fas +regressions/ck_spinlock/validate/ck_hclh regressions/ck_spinlock/validate/ck_mcs +regressions/ck_spinlock/validate/ck_spinlock regressions/ck_spinlock/validate/ck_ticket regressions/ck_spinlock/validate/ck_ticket_pb regressions/ck_spinlock/validate/linux_spinlock @@ -132,6 +154,9 @@ regressions/ck_stack/benchmark/latency regressions/ck_stack/validate/mpmc_pair regressions/ck_stack/validate/mpmc_pop regressions/ck_stack/validate/mpmc_push +regressions/ck_stack/validate/mpmc_trypair +regressions/ck_stack/validate/mpmc_trypop +regressions/ck_stack/validate/mpmc_trypush regressions/ck_stack/validate/mpnc_push regressions/ck_stack/validate/pthreads_pair regressions/ck_stack/validate/serial @@ -143,33 +168,8 @@ regressions/ck_stack/validate/spinlock_pop regressions/ck_stack/validate/spinlock_push regressions/ck_stack/validate/upmc_pop regressions/ck_stack/validate/upmc_push -regressions/ck_brlock/benchmark/throughput -regressions/ck_rwlock/benchmark/throughput -regressions/ck_queue/validate/ck_list -regressions/ck_queue/validate/ck_slist -regressions/ck_queue/validate/ck_stailq -regressions/ck_cohort/validate/validate -regressions/ck_cohort/benchmark/ck_cohort.LATENCY -regressions/ck_cohort/benchmark/ck_cohort.THROUGHPUT -regressions/ck_rwcohort/validate/ck_neutral -regressions/ck_rwcohort/validate/ck_rp -regressions/ck_rwcohort/validate/ck_wp -regressions/ck_rwcohort/benchmark/ck_neutral.LATENCY -regressions/ck_rwcohort/benchmark/ck_neutral.THROUGHPUT -regressions/ck_rwcohort/benchmark/ck_rp.LATENCY -regressions/ck_rwcohort/benchmark/ck_rp.THROUGHPUT -regressions/ck_rwcohort/benchmark/ck_wp.LATENCY -regressions/ck_rwcohort/benchmark/ck_wp.THROUGHPUT -regressions/ck_hs/benchmark/parallel_bytestring.delete -regressions/ck_ht/benchmark/parallel_bytestring.delete -regressions/ck_ht/benchmark/serial.delete -regressions/ck_ht/validate/serial.delete -regressions/ck_rhs/benchmark/parallel_bytestring -regressions/ck_rhs/benchmark/serial -regressions/ck_rhs/validate/serial -regressions/ck_spinlock/benchmark/ck_hclh.LATENCY -regressions/ck_spinlock/benchmark/ck_hclh.THROUGHPUT -regressions/ck_spinlock/validate/ck_hclh +regressions/ck_stack/validate/upmc_trypop +regressions/ck_stack/validate/upmc_trypush regressions/ck_swlock/benchmark/latency regressions/ck_swlock/benchmark/throughput regressions/ck_swlock/validate/validate diff --git a/include/ck_bitmap.h b/include/ck_bitmap.h index 475caef..8db09dd 100644 --- a/include/ck_bitmap.h +++ b/include/ck_bitmap.h @@ -1,6 +1,7 @@ /* * Copyright 2012-2014 Samy Al Bahra * Copyright 2012-2014 AppNexus, Inc. + * Copyright 2014 Paul Khuong. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,37 +37,22 @@ #include #include -#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_STORE_64) && \ - defined(CK_F_PR_AND_64) && defined(CK_F_PR_OR_64) -#define CK_BITMAP_WORD uint64_t -#define CK_BITMAP_SHIFT 6 -#define CK_BITMAP_STORE(x, y) ck_pr_store_64(x, y) -#define CK_BITMAP_LOAD(x) ck_pr_load_64(x) -#define CK_BITMAP_OR(x, y) ck_pr_or_64(x, y) -#define CK_BITMAP_AND(x, y) ck_pr_and_64(x, y) -#elif defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_STORE_32) && \ - defined(CK_F_PR_AND_32) && defined(CK_F_PR_OR_32) -#define CK_BITMAP_WORD uint32_t -#define CK_BITMAP_SHIFT 5 -#define CK_BITMAP_STORE(x, y) ck_pr_store_32(x, y) -#define CK_BITMAP_LOAD(x) ck_pr_load_32(x) -#define CK_BITMAP_OR(x, y) ck_pr_or_32(x, y) -#define CK_BITMAP_AND(x, y) ck_pr_and_32(x, y) -#else +#if !defined(CK_F_PR_LOAD_UINT) || !defined(CK_F_PR_STORE_UINT) || \ + !defined(CK_F_PR_AND_UINT) || !defined(CK_F_PR_OR_UINT) || \ + !defined(CK_F_CC_CTZ) #error "ck_bitmap is not supported on your platform." -#endif /* These are all internal functions. */ +#endif -#define CK_BITMAP_PTR(x, i) ((x) + ((i) >> CK_BITMAP_SHIFT)) -#define CK_BITMAP_BLOCK (sizeof(CK_BITMAP_WORD) * CHAR_BIT) -#define CK_BITMAP_MASK (CK_BITMAP_BLOCK - 1) -#define CK_BITMAP_BLOCKS(n) \ - (((n) + (CK_BITMAP_BLOCK - 1)) / CK_BITMAP_BLOCK) +#define CK_BITMAP_BLOCK (sizeof(unsigned int) * CHAR_BIT) +#define CK_BITMAP_BIT(i) (1U << ((i) % CK_BITMAP_BLOCK)) +#define CK_BITMAP_PTR(x, i) ((x) + ((i) / CK_BITMAP_BLOCK)) +#define CK_BITMAP_BLOCKS(n) (((n) + CK_BITMAP_BLOCK - 1) / CK_BITMAP_BLOCK) #define CK_BITMAP_INSTANCE(n_entries) \ union { \ struct { \ unsigned int n_bits; \ - CK_BITMAP_WORD map[CK_BITMAP_BLOCKS(n_entries)];\ + unsigned int map[CK_BITMAP_BLOCKS(n_entries)]; \ } content; \ struct ck_bitmap bitmap; \ } @@ -83,17 +69,35 @@ #define CK_BITMAP_SET(a, b) \ ck_bitmap_set(&(a)->bitmap, (b)) +#define CK_BITMAP_RESET(a, b) \ + ck_bitmap_reset(&(a)->bitmap, (b)) + +#define CK_BITMAP_TEST(a, b) \ + ck_bitmap_test(&(a)->bitmap, (b)) + #define CK_BITMAP_UNION(a, b) \ ck_bitmap_union(&(a)->bitmap, &(b)->bitmap) -#define CK_BITMAP_RESET(a, b) \ - ck_bitmap_reset(&(a)->bitmap, (b)) +#define CK_BITMAP_INTERSECTION(a, b) \ + ck_bitmap_intersection(&(a)->bitmap, &(b)->bitmap) + +#define CK_BITMAP_INTERSECTION_NEGATE(a, b) \ + ck_bitmap_intersection_negate(&(a)->bitmap, &(b)->bitmap) #define CK_BITMAP_CLEAR(a) \ ck_bitmap_clear(&(a)->bitmap) -#define CK_BITMAP_TEST(a, b) \ - ck_bitmap_test(&(a)->bitmap, (b)) +#define CK_BITMAP_EMPTY(a, b) \ + ck_bitmap_empty(&(a)->bitmap, b) + +#define CK_BITMAP_FULL(a, b) \ + ck_bitmap_full(&(a)->bitmap, b) + +#define CK_BITMAP_COUNT(a, b) \ + ck_bitmap_count(&(a)->bitmap, b) + +#define CK_BITMAP_COUNT_INTERSECT(a, b, c) \ + ck_bitmap_count_intersect(&(a)->bitmap, b, c) #define CK_BITMAP_BITS(a) \ ck_bitmap_bits(&(a)->bitmap) @@ -106,14 +110,13 @@ struct ck_bitmap { unsigned int n_bits; - CK_BITMAP_WORD map[]; + unsigned int map[]; }; typedef struct ck_bitmap ck_bitmap_t; struct ck_bitmap_iterator { - CK_BITMAP_WORD cache; + unsigned int cache; unsigned int n_block; - unsigned int n_bit; unsigned int n_limit; }; typedef struct ck_bitmap_iterator ck_bitmap_iterator_t; @@ -122,7 +125,7 @@ CK_CC_INLINE static unsigned int ck_bitmap_base(unsigned int n_bits) { - return CK_BITMAP_BLOCKS(n_bits) * sizeof(CK_BITMAP_WORD); + return CK_BITMAP_BLOCKS(n_bits) * sizeof(unsigned int); } /* @@ -136,24 +139,68 @@ ck_bitmap_size(unsigned int n_bits) return ck_bitmap_base(n_bits) + sizeof(struct ck_bitmap); } +/* + * Returns total number of bits in specified bitmap. + */ +CK_CC_INLINE static unsigned int +ck_bitmap_bits(const struct ck_bitmap *bitmap) +{ + + return bitmap->n_bits; +} + +/* + * Returns a pointer to the bit buffer associated + * with the specified bitmap. + */ +CK_CC_INLINE static void * +ck_bitmap_buffer(struct ck_bitmap *bitmap) +{ + + return bitmap->map; +} + /* * Sets the bit at the offset specified in the second argument. */ CK_CC_INLINE static void ck_bitmap_set(struct ck_bitmap *bitmap, unsigned int n) { - CK_BITMAP_WORD mask = 0x1ULL << (n & CK_BITMAP_MASK); - CK_BITMAP_OR(CK_BITMAP_PTR(bitmap->map, n), mask); + ck_pr_or_uint(CK_BITMAP_PTR(bitmap->map, n), CK_BITMAP_BIT(n)); + return; +} + +/* + * Resets the bit at the offset specified in the second argument. + */ +CK_CC_INLINE static void +ck_bitmap_reset(struct ck_bitmap *bitmap, unsigned int n) +{ + + ck_pr_and_uint(CK_BITMAP_PTR(bitmap->map, n), ~CK_BITMAP_BIT(n)); return; } +/* + * Determines whether the bit at offset specified in the + * second argument is set. + */ +CK_CC_INLINE static bool +ck_bitmap_test(const struct ck_bitmap *bitmap, unsigned int n) +{ + unsigned int block; + + block = ck_pr_load_uint(CK_BITMAP_PTR(bitmap->map, n)); + return block & CK_BITMAP_BIT(n); +} + /* * Combines bits from second bitmap into the first bitmap. This is not a * linearized operation with respect to the complete bitmap. */ CK_CC_INLINE static void -ck_bitmap_union(struct ck_bitmap *dst, struct ck_bitmap *src) +ck_bitmap_union(struct ck_bitmap *dst, const struct ck_bitmap *src) { unsigned int n; unsigned int n_buckets = dst->n_bits; @@ -162,21 +209,62 @@ ck_bitmap_union(struct ck_bitmap *dst, struct ck_bitmap *src) n_buckets = src->n_bits; n_buckets = CK_BITMAP_BLOCKS(n_buckets); - for (n = 0; n < n_buckets; n++) - CK_BITMAP_OR(&dst->map[n], src->map[n]); + for (n = 0; n < n_buckets; n++) { + ck_pr_or_uint(&dst->map[n], + ck_pr_load_uint(&src->map[n])); + } return; } /* - * Resets the bit at the offset specified in the second argument. + * Intersects bits from second bitmap into the first bitmap. This is + * not a linearized operation with respect to the complete bitmap. + * Any trailing bit in dst is cleared. */ CK_CC_INLINE static void -ck_bitmap_reset(struct ck_bitmap *bitmap, unsigned int n) +ck_bitmap_intersection(struct ck_bitmap *dst, const struct ck_bitmap *src) +{ + unsigned int n; + unsigned int n_buckets = dst->n_bits; + unsigned int n_intersect = n_buckets; + + if (src->n_bits < n_intersect) + n_intersect = src->n_bits; + + n_buckets = CK_BITMAP_BLOCKS(n_buckets); + n_intersect = CK_BITMAP_BLOCKS(n_intersect); + for (n = 0; n < n_intersect; n++) { + ck_pr_and_uint(&dst->map[n], + ck_pr_load_uint(&src->map[n])); + } + + for (; n < n_buckets; n++) + ck_pr_store_uint(&dst->map[n], 0); + + return; +} + +/* + * Intersects the complement of bits from second bitmap into the first + * bitmap. This is not a linearized operation with respect to the + * complete bitmap. Any trailing bit in dst is left as is. + */ +CK_CC_INLINE static void +ck_bitmap_intersection_negate(struct ck_bitmap *dst, const struct ck_bitmap *src) { - CK_BITMAP_WORD mask = ~(0x1ULL << (n & CK_BITMAP_MASK)); + unsigned int n; + unsigned int n_intersect = dst->n_bits; + + if (src->n_bits < n_intersect) + n_intersect = src->n_bits; + + n_intersect = CK_BITMAP_BLOCKS(n_intersect); + for (n = 0; n < n_intersect; n++) { + ck_pr_and_uint(&dst->map[n], + (~ck_pr_load_uint(&src->map[n]))); + } - CK_BITMAP_AND(CK_BITMAP_PTR(bitmap->map, n), mask); return; } @@ -187,48 +275,139 @@ ck_bitmap_reset(struct ck_bitmap *bitmap, unsigned int n) CK_CC_INLINE static void ck_bitmap_clear(struct ck_bitmap *bitmap) { - unsigned int n_buckets = ck_bitmap_base(bitmap->n_bits) / sizeof(CK_BITMAP_WORD); + unsigned int n_buckets = ck_bitmap_base(bitmap->n_bits) / sizeof(unsigned int); unsigned int i; for (i = 0; i < n_buckets; i++) - CK_BITMAP_STORE(&bitmap->map[i], 0); + ck_pr_store_uint(&bitmap->map[i], 0); return; } /* - * Determines whether the bit at offset specified in the - * second argument is set. + * Returns true if the first limit bits in bitmap are cleared. If + * limit is greater than the bitmap size, limit is truncated to that + * size. */ CK_CC_INLINE static bool -ck_bitmap_test(struct ck_bitmap *bitmap, unsigned int n) +ck_bitmap_empty(const ck_bitmap_t *bitmap, unsigned int limit) { - CK_BITMAP_WORD mask = 0x1ULL << (n & CK_BITMAP_MASK); - CK_BITMAP_WORD block; + unsigned int i, words, slop; + + if (limit > bitmap->n_bits) + limit = bitmap->n_bits; + + words = limit / CK_BITMAP_BLOCK; + slop = limit % CK_BITMAP_BLOCK; + for (i = 0; i < words; i++) { + if (ck_pr_load_uint(&bitmap->map[i]) != 0) { + return false; + } + } + + if (slop > 0) { + unsigned int word; + + word = ck_pr_load_uint(&bitmap->map[i]); + if ((word & ((1U << slop) - 1)) != 0) + return false; + } - block = CK_BITMAP_LOAD(CK_BITMAP_PTR(bitmap->map, n)); - return block & mask; + return true; } /* - * Returns total number of bits in specified bitmap. + * Returns true if the first limit bits in bitmap are set. If limit + * is greater than the bitmap size, limit is truncated to that size. + */ +CK_CC_UNUSED static bool +ck_bitmap_full(const ck_bitmap_t *bitmap, unsigned int limit) +{ + unsigned int i, slop, words; + + if (limit > bitmap->n_bits) { + limit = bitmap->n_bits; + } + + words = limit / CK_BITMAP_BLOCK; + slop = limit % CK_BITMAP_BLOCK; + for (i = 0; i < words; i++) { + if (ck_pr_load_uint(&bitmap->map[i]) != -1U) + return false; + } + + if (slop > 0) { + unsigned int word; + + word = ~ck_pr_load_uint(&bitmap->map[i]); + if ((word & ((1U << slop) - 1)) != 0) + return false; + } + return true; +} + +/* + * Returns the number of set bit in bitmap, upto (and excluding) + * limit. If limit is greater than the bitmap size, it is truncated + * to that size. */ CK_CC_INLINE static unsigned int -ck_bitmap_bits(struct ck_bitmap *bitmap) +ck_bitmap_count(const ck_bitmap_t *bitmap, unsigned int limit) { + unsigned int count, i, slop, words; - return bitmap->n_bits; + if (limit > bitmap->n_bits) + limit = bitmap->n_bits; + + words = limit / CK_BITMAP_BLOCK; + slop = limit % CK_BITMAP_BLOCK; + for (i = 0, count = 0; i < words; i++) + count += ck_cc_popcount(ck_pr_load_uint(&bitmap->map[i])); + + if (slop > 0) { + unsigned int word; + + word = ck_pr_load_uint(&bitmap->map[i]); + count += ck_cc_popcount(word & ((1U << slop) - 1)); + } + return count; } /* - * Returns a pointer to the bit buffer associated - * with the specified bitmap. + * Returns the number of set bit in the intersection of two bitmaps, + * upto (and exclusing) limit. If limit is greater than either bitmap + * size, it is truncated to the smallest. */ -CK_CC_INLINE static void * -ck_bitmap_buffer(struct ck_bitmap *bitmap) +CK_CC_INLINE static unsigned int +ck_bitmap_count_intersect(const ck_bitmap_t *x, const ck_bitmap_t *y, unsigned int limit) { + unsigned int count, i, slop, words; - return bitmap->map; + if (limit > x->n_bits) + limit = x->n_bits; + + if (limit > y->n_bits) + limit = y->n_bits; + + words = limit / CK_BITMAP_BLOCK; + slop = limit % CK_BITMAP_BLOCK; + for (i = 0, count = 0; i < words; i++) { + unsigned int xi, yi; + + xi = ck_pr_load_uint(&x->map[i]); + yi = ck_pr_load_uint(&y->map[i]); + count += ck_cc_popcount(xi & yi); + } + + if (slop > 0) { + unsigned int word, xi, yi; + + xi = ck_pr_load_uint(&x->map[i]); + yi = ck_pr_load_uint(&y->map[i]); + word = xi & yi; + count += ck_cc_popcount(word & ((1U << slop) - 1)); + } + return count; } /* @@ -247,34 +426,31 @@ ck_bitmap_init(struct ck_bitmap *bitmap, memset(bitmap->map, -(int)set, base); if (set == true) { - CK_BITMAP_WORD b; - - if (n_bits < CK_BITMAP_BLOCK) - b = n_bits; - else - b = n_bits % CK_BITMAP_BLOCK; + unsigned int b = n_bits % CK_BITMAP_BLOCK; if (b == 0) return; - bitmap->map[base / sizeof(CK_BITMAP_WORD) - 1] &= (1ULL << b) - 1ULL; + *CK_BITMAP_PTR(bitmap->map, n_bits - 1) &= (1U << b) - 1U; } return; } - /* * Initialize iterator for use with provided bitmap. */ CK_CC_INLINE static void -ck_bitmap_iterator_init(struct ck_bitmap_iterator *i, struct ck_bitmap *bitmap) +ck_bitmap_iterator_init(struct ck_bitmap_iterator *i, const struct ck_bitmap *bitmap) { i->n_block = 0; - i->n_bit = 0; i->n_limit = CK_BITMAP_BLOCKS(bitmap->n_bits); - i->cache = CK_BITMAP_LOAD(&bitmap->map[i->n_block]); + if (i->n_limit > 0) { + i->cache = ck_pr_load_uint(&bitmap->map[0]); + } else { + i->cache = 0; + } return; } @@ -282,38 +458,34 @@ ck_bitmap_iterator_init(struct ck_bitmap_iterator *i, struct ck_bitmap *bitmap) * Iterate to next bit. */ CK_CC_INLINE static bool -ck_bitmap_next(struct ck_bitmap *bitmap, +ck_bitmap_next(const struct ck_bitmap *bitmap, struct ck_bitmap_iterator *i, unsigned int *bit) { + unsigned int cache = i->cache; + unsigned int n_block = i->n_block; + unsigned int n_limit = i->n_limit; - /* Load next bitmap block. */ - for (;;) { - while (i->n_bit < CK_BITMAP_BLOCK) { - unsigned int previous = i->n_bit++; - - if (i->cache & 1) { - *bit = previous + (CK_BITMAP_BLOCK * i->n_block); - i->cache >>= 1; - return true; - } + if (cache == 0) { + if (n_block >= n_limit) + return false; - i->cache >>= 1; - if (i->cache == 0) - break; + for (n_block++; n_block < n_limit; n_block++) { + cache = ck_pr_load_uint(&bitmap->map[n_block]); + if (cache != 0) + goto non_zero; } - i->n_bit = 0; - i->n_block++; - - if (i->n_block >= i->n_limit) - return false; - - i->cache = CK_BITMAP_LOAD(&bitmap->map[i->n_block]); + i->cache = 0; + i->n_block = n_block; + return false; } - return false; +non_zero: + *bit = CK_BITMAP_BLOCK * n_block + ck_cc_ctz(cache); + i->cache = cache & (cache - 1); + i->n_block = n_block; + return true; } #endif /* _CK_BITMAP_H */ - diff --git a/include/ck_cc.h b/include/ck_cc.h index c85176d..a4baaa5 100644 --- a/include/ck_cc.h +++ b/include/ck_cc.h @@ -1,5 +1,6 @@ /* * Copyright 2009-2014 Samy Al Bahra. + * Copyright 2014 Paul Khuong. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -85,5 +86,70 @@ #define CK_CC_UNLIKELY(x) x #endif -#endif /* _CK_CC_H */ +#ifndef CK_F_CC_FFS +#define CK_F_CC_FFS +CK_CC_INLINE static int +ck_cc_ffs(unsigned int x) +{ + unsigned int i; + + if (x == 0) + return 0; + + for (i = 1; (x & 1) == 0; i++, x >>= 1); + + return i; +} +#endif + +#ifndef CK_F_CC_CLZ +#define CK_F_CC_CLZ +#include + +CK_CC_INLINE static int +ck_cc_clz(unsigned int x) +{ + unsigned int count, i; + + for (count = 0, i = sizeof(unsigned int) * CHAR_BIT; i > 0; count++) { + unsigned int bit = 1U << --i; + + if (x & bit) + break; + } + + return count; +} +#endif + +#ifndef CK_F_CC_CTZ +#define CK_F_CC_CTZ +CK_CC_INLINE static int +ck_cc_ctz(unsigned int x) +{ + unsigned int i; + + if (x == 0) + return 0; + + for (i = 0; (x & 1) == 0; i++, x >>= 1); + + return i; +} +#endif +#ifndef CK_F_CC_POPCOUNT +#define CK_F_CC_POPCOUNT +CK_CC_INLINE static int +ck_cc_popcount(unsigned int x) +{ + unsigned int acc; + + for (acc = 0; x != 0; x >>= 1) + acc += x & 1; + + return acc; +} +#endif + +#endif /* _CK_CC_H */ diff --git a/include/gcc/ck_cc.h b/include/gcc/ck_cc.h index 7a98c94..084e7bf 100644 --- a/include/gcc/ck_cc.h +++ b/include/gcc/ck_cc.h @@ -1,5 +1,6 @@ /* * Copyright 2009-2014 Samy Al Bahra. + * Copyright 2014 Paul Khuong. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -96,5 +97,41 @@ */ #define CK_CC_ALIASED __attribute__((__may_alias__)) -#endif /* _CK_GCC_CC_H */ +/* + * Portability wrappers for bitwise ops. + */ + +#define CK_F_CC_FFS +#define CK_F_CC_CLZ +#define CK_F_CC_CTZ +#define CK_F_CC_POPCOUNT + +CK_CC_INLINE static int +ck_cc_ffs(unsigned int x) +{ + + return __builtin_ffs(x); +} + +CK_CC_INLINE static int +ck_cc_clz(unsigned int x) +{ + return __builtin_clz(x); +} + +CK_CC_INLINE static int +ck_cc_ctz(unsigned int x) +{ + + return __builtin_ctz(x); +} + +CK_CC_INLINE static int +ck_cc_popcount(unsigned int x) +{ + + return __builtin_popcount(x); +} + +#endif /* _CK_GCC_CC_H */ diff --git a/regressions/ck_bitmap/validate/serial.c b/regressions/ck_bitmap/validate/serial.c index 57230a5..79aca8d 100644 --- a/regressions/ck_bitmap/validate/serial.c +++ b/regressions/ck_bitmap/validate/serial.c @@ -2,6 +2,7 @@ * Copyright 2012-2014 Samy Al Bahra. * Copyright 2012-2014 AppNexus, Inc. * Copyright 2012 Shreyas Prasad. + * Copyright 2014 Paul Khuong. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -127,10 +128,171 @@ test(ck_bitmap_t *bits, unsigned int n_length, bool initial) return; } +static void +test_init(bool init) +{ + ck_bitmap_t *bitmap; + size_t bytes; + unsigned int i; + + bytes = ck_bitmap_size(length); + bitmap = malloc(bytes); + memset(bitmap, random(), bytes); + + ck_bitmap_init(bitmap, length, init); + + if (ck_bitmap_bits(bitmap) != length) { + ck_error("ERROR: Expected length %u got %u\n", + length, ck_bitmap_bits(bitmap)); + } + + for (i = 0; i < length; i++) { + if (ck_bitmap_test(bitmap, i) != init) { + ck_error("ERROR: Expected bit %i at index %u, got %i\n", + (int)init, i, (int)(!init)); + } + } + + free(bitmap); +} + +static ck_bitmap_t * +random_init(void) +{ + ck_bitmap_t *bitmap; + unsigned int i; + + bitmap = malloc(ck_bitmap_size(length)); + ck_bitmap_init(bitmap, length, false); + + for (i = 0; i < length; i++) { + if (random() & 1) { + ck_bitmap_set(bitmap, i); + } + } + + return bitmap; +} + +static ck_bitmap_t * +copy(const ck_bitmap_t *src) +{ + ck_bitmap_t *bitmap; + size_t bytes = ck_bitmap_size(ck_bitmap_bits(src)); + + bitmap = malloc(bytes); + memcpy(bitmap, src, bytes); + return bitmap; +} + +static void +test_counts(const ck_bitmap_t *x, const ck_bitmap_t *y) +{ + unsigned int count = 0; + unsigned int count_intersect = 0; + unsigned int i; + + for (i = 0; i <= length * 2; i++) { + unsigned actual_limit = i; + unsigned int r; + bool check; + + if (actual_limit > ck_bitmap_bits(x)) + actual_limit = ck_bitmap_bits(x); + + check = ck_bitmap_empty(x, i); + if (check != (count == 0)) { + ck_error("ck_bitmap_empty(%u): got %i expected %i\n", + i, (int)check, (int)(count == 0)); + } + + check = ck_bitmap_full(x, i); + if (check != (count == actual_limit)) { + ck_error("ck_bitmap_full(%u): got %i expected %i\n", + i, (int)check, (int)(count == i)); + } + + r = ck_bitmap_count(x, i); + if (r != count) { + ck_error("ck_bitmap_count(%u): got %u expected %u\n", + i, r, count); + } + + r = ck_bitmap_count_intersect(x, y, i); + if (r != count_intersect) { + ck_error("ck_bitmap_count_intersect(%u): got %u expected %u\n", + i, r, count_intersect); + } + + if (i < length) { + count += ck_bitmap_test(x, i); + count_intersect += ck_bitmap_test(x, i) & ck_bitmap_test(y, i); + } + } +} + +static void +random_test(unsigned int seed) +{ + ck_bitmap_t *x, *x_copy, *y; + unsigned int i; + + srandom(seed); + + test_init(false); + test_init(true); + + x = random_init(); + y = random_init(); + +#define TEST(routine, expected) do { \ + x_copy = copy(x); \ + routine(x_copy, y); \ + for (i = 0; i < length; i++) { \ + bool xi = ck_bitmap_test(x, i); \ + bool yi = ck_bitmap_test(y, i); \ + bool ri = ck_bitmap_test(x_copy, i); \ + bool wanted = expected(xi, yi); \ + \ + if (ri != wanted) { \ + ck_error("In " #routine " at %u: " \ + "got %i expected %i\n", \ + i, ri, wanted); \ + } \ + } \ + free(x_copy); \ + } while (0) + +#define OR(x, y) (x | y) +#define AND(x, y) (x & y) +#define ANDC2(x, y) (x & (~y)) + + TEST(ck_bitmap_union, OR); + TEST(ck_bitmap_intersection, AND); + TEST(ck_bitmap_intersection_negate, ANDC2); + +#undef ANDC2 +#undef AND +#undef OR +#undef TEST + + test_counts(x, y); + + for (i = 0; i < 4; i++) { + ck_bitmap_init(x, length, i & 1); + ck_bitmap_init(y, length, i >> 1); + test_counts(x, y); + } + + free(y); + free(x); +} + int main(int argc, char *argv[]) { unsigned int bytes, base; + size_t i, j; if (argc >= 2) { length = atoi(argv[1]); @@ -177,6 +339,12 @@ main(int argc, char *argv[]) ck_error("ERROR: Expected bit to be reset.\n"); } + for (i = 0; i < 4 * sizeof(unsigned int) * CHAR_BIT; i++) { + length = i; + for (j = 0; j < 10; j++) { + random_test(i * 10 + j); + } + } + return 0; } - diff --git a/regressions/ck_rwlock/validate/Makefile b/regressions/ck_rwlock/validate/Makefile index 608416c..2c2116b 100644 --- a/regressions/ck_rwlock/validate/Makefile +++ b/regressions/ck_rwlock/validate/Makefile @@ -14,4 +14,4 @@ clean: rm -rf *.dSYM *.exe *~ *.o $(OBJECTS) include ../../../build/regressions.build -CFLAGS+=$(PTHREAD_CFLAGS) -D_GNU_SOURCE -O0 +CFLAGS+=$(PTHREAD_CFLAGS) -D_GNU_SOURCE