diff --git a/include/ck_brlock.h b/include/ck_brlock.h new file mode 100644 index 0000000..50a4f5e --- /dev/null +++ b/include/ck_brlock.h @@ -0,0 +1,169 @@ +/* + * Copyright 2011 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _CK_BRLOCK_H +#define _CK_BRLOCK_H + +/* + * Big reader spinlocks provide cache-local contention-free read + * lock acquisition in the absence of writers. This comes at the + * cost of O(n) write lock acquisition. They were first implemented + * in the Linux kernel by Ingo Molnar and David S. Miller around the + * year 2000. + * + * This implementation is thread-agnostic which comes at the cost + * of larger reader objects due to necessary linkage overhead. In + * order to cut down on TLB pressure, it is recommended to allocate + * these objects on the same page. + */ + +#include +#include +#include + +struct ck_brlock_reader { + unsigned int n_readers; + struct ck_brlock_reader *previous; + struct ck_brlock_reader *next; +}; +typedef struct ck_brlock_reader ck_brlock_reader_t; + +#define CK_BRLOCK_READER_INITIALIZER {0} + +struct ck_brlock { + struct ck_brlock_reader *readers; + unsigned int writer; +}; +typedef struct ck_brlock ck_brlock_t; + +#define CK_BRLOCK_INITIALIZER {NULL, false} + +CK_CC_INLINE static void +ck_brlock_init(struct ck_brlock *br) +{ + + br->readers = NULL; + br->writer = false; + ck_pr_fence_memory(); + return; +} + +CK_CC_INLINE static void +ck_brlock_write_lock(struct ck_brlock *br) +{ + struct ck_brlock_reader *cursor; + + /* + * As the frequency of write acquisitions should be low, + * there is no point to more advanced contention avoidance. + */ + while (ck_pr_fas_uint(&br->writer, true) == true) + ck_pr_stall(); + + /* The reader list is protected under the writer br. */ + for (cursor = br->readers; cursor != NULL; cursor = cursor->next) { + while (ck_pr_load_uint(&cursor->n_readers) != 0) + ck_pr_stall(); + } + + /* This branch should never be reached. */ + return; +} + +CK_CC_INLINE static void +ck_brlock_write_unlock(struct ck_brlock *br) +{ + + ck_pr_fence_memory(); + ck_pr_store_uint(&br->writer, false); + return; +} + +CK_CC_INLINE static void +ck_brlock_read_register(struct ck_brlock *br, struct ck_brlock_reader *reader) +{ + + reader->n_readers = 0; + reader->previous = NULL; + + /* Implicit compiler barrier. */ + ck_brlock_write_lock(br); + + reader->next = ck_pr_load_ptr(&br->readers); + if (reader->next != NULL) + reader->next->previous = reader; + ck_pr_store_ptr(&br->readers, reader); + + ck_brlock_write_unlock(br); + return; +} + +CK_CC_INLINE static void +ck_brlock_read_unregister(struct ck_brlock *br, struct ck_brlock_reader *reader) +{ + + ck_brlock_write_lock(br); + + if (reader->next != NULL) + reader->next->previous = reader->previous; + + if (reader->previous != NULL) + reader->previous->next = reader->next; + else + br->readers = reader->next; + + ck_brlock_write_unlock(br); + return; +} + +CK_CC_INLINE static void +ck_brlock_read_lock(struct ck_brlock *br, struct ck_brlock_reader *reader) +{ + + for (;;) { + while (ck_pr_load_uint(&br->writer) == true) + ck_pr_stall(); + + ck_pr_inc_uint(&reader->n_readers); + + if (ck_pr_load_uint(&br->writer) == false) + break; + + ck_pr_dec_uint(&reader->n_readers); + } + + return; +} + +CK_CC_INLINE static void +ck_brlock_read_unlock(struct ck_brlock_reader *reader) +{ + + ck_pr_dec_uint(&reader->n_readers); + return; +} + +#endif /* _CK_BRLOCK_H */ diff --git a/regressions/ck_brlock/benchmark/latency.c b/regressions/ck_brlock/benchmark/latency.c new file mode 100644 index 0000000..2d9c63f --- /dev/null +++ b/regressions/ck_brlock/benchmark/latency.c @@ -0,0 +1,165 @@ +/* + * Copyright 2011 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "../../common.h" + +#ifndef STEPS +#define STEPS 1000000 +#endif + +/* + * This is a naive reader/writer spinlock. + */ +struct rwlock { + unsigned int readers; + ck_spinlock_fas_t writer; +}; +typedef struct rwlock rwlock_t; + +static CK_CC_INLINE void +rwlock_init(rwlock_t *rw) +{ + + ck_pr_store_uint(&rw->readers, 0); + ck_spinlock_fas_init(&rw->writer); + return; +} + +static CK_CC_INLINE void +rwlock_write_lock(rwlock_t *rw) +{ + + ck_spinlock_fas_lock(&rw->writer); + while (ck_pr_load_uint(&rw->readers) != 0) + ck_pr_stall(); + + return; +} + +static CK_CC_INLINE void +rwlock_write_unlock(rwlock_t *rw) +{ + + ck_spinlock_fas_unlock(&rw->writer); + return; +} + +static CK_CC_INLINE void +rwlock_read_lock(rwlock_t *rw) +{ + + for (;;) { + while (ck_pr_load_uint(&rw->writer.value) != 0) + ck_pr_stall(); + + ck_pr_inc_uint(&rw->readers); + if (ck_pr_load_uint(&rw->writer.value) == 0) + break; + ck_pr_dec_uint(&rw->readers); + } + + return; +} + +static CK_CC_INLINE void +rwlock_read_unlock(rwlock_t *rw) +{ + + ck_pr_dec_uint(&rw->readers); + return; +} + +int +main(void) +{ + uint64_t s_b, e_b, i; + ck_brlock_t brlock = CK_BRLOCK_INITIALIZER; + ck_brlock_reader_t r[8]; + rwlock_t naive; + + for (i = 0; i < sizeof(r) / sizeof(*r); i++) + ck_brlock_read_register(&brlock, &r[i]); + + for (i = 0; i < STEPS; i++) { + ck_brlock_write_lock(&brlock); + ck_brlock_write_unlock(&brlock); + } + + s_b = rdtsc(); + for (i = 0; i < STEPS; i++) { + ck_brlock_write_lock(&brlock); + ck_brlock_write_unlock(&brlock); + } + e_b = rdtsc(); + printf("WRITE: brlock %15" PRIu64 "\n", (e_b - s_b) / STEPS); + + rwlock_init(&naive); + for (i = 0; i < STEPS; i++) { + rwlock_write_lock(&naive); + rwlock_write_unlock(&naive); + } + + s_b = rdtsc(); + for (i = 0; i < STEPS; i++) { + rwlock_write_lock(&naive); + rwlock_write_unlock(&naive); + } + e_b = rdtsc(); + printf("WRITE: naive %15" PRIu64 "\n", (e_b - s_b) / STEPS); + + for (i = 0; i < STEPS; i++) { + ck_brlock_read_lock(&brlock, &r[0]); + ck_brlock_read_unlock(&r[0]); + } + s_b = rdtsc(); + for (i = 0; i < STEPS; i++) { + ck_brlock_read_lock(&brlock, &r[0]); + ck_brlock_read_unlock(&r[0]); + } + e_b = rdtsc(); + printf("READ: brlock %15" PRIu64 "\n", (e_b - s_b) / STEPS); + + for (i = 0; i < STEPS; i++) { + rwlock_read_lock(&naive); + rwlock_read_unlock(&naive); + } + + s_b = rdtsc(); + for (i = 0; i < STEPS; i++) { + rwlock_read_lock(&naive); + rwlock_read_unlock(&naive); + } + e_b = rdtsc(); + printf("READ: naive %15" PRIu64 "\n", (e_b - s_b) / STEPS); + + return (0); +} + diff --git a/regressions/ck_brlock/validate/validate.c b/regressions/ck_brlock/validate/validate.c new file mode 100644 index 0000000..bd372f4 --- /dev/null +++ b/regressions/ck_brlock/validate/validate.c @@ -0,0 +1,163 @@ +/* + * Copyright 2011 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "../../common.h" + +#ifndef ITERATE +#define ITERATE 5000000 +#endif + +static struct affinity a; +static unsigned int locked = 0; +static int nthr; +static ck_brlock_t lock = CK_BRLOCK_INITIALIZER; + +static void * +thread(void *null CK_CC_UNUSED) +{ + ck_brlock_reader_t r; + int i = ITERATE; + unsigned int l; + + if (aff_iterate(&a)) { + perror("ERROR: Could not affine thread"); + exit(EXIT_FAILURE); + } + + ck_brlock_read_register(&lock, &r); + + while (i--) { + ck_brlock_write_lock(&lock); + { + l = ck_pr_load_uint(&locked); + if (l != 0) { + fprintf(stderr, "ERROR [WR:%d]: %u != 0\n", __LINE__, l); + exit(EXIT_FAILURE); + } + + ck_pr_inc_uint(&locked); + ck_pr_inc_uint(&locked); + ck_pr_inc_uint(&locked); + ck_pr_inc_uint(&locked); + ck_pr_inc_uint(&locked); + ck_pr_inc_uint(&locked); + ck_pr_inc_uint(&locked); + ck_pr_inc_uint(&locked); + + l = ck_pr_load_uint(&locked); + if (l != 8) { + fprintf(stderr, "ERROR [WR:%d]: %u != 2\n", __LINE__, l); + exit(EXIT_FAILURE); + } + + ck_pr_dec_uint(&locked); + ck_pr_dec_uint(&locked); + ck_pr_dec_uint(&locked); + ck_pr_dec_uint(&locked); + ck_pr_dec_uint(&locked); + ck_pr_dec_uint(&locked); + ck_pr_dec_uint(&locked); + ck_pr_dec_uint(&locked); + + l = ck_pr_load_uint(&locked); + if (l != 0) { + fprintf(stderr, "ERROR [WR:%d]: %u != 0\n", __LINE__, l); + exit(EXIT_FAILURE); + } + } + ck_brlock_write_unlock(&lock); + + ck_brlock_read_lock(&lock, &r); + { + l = ck_pr_load_uint(&locked); + if (l != 0) { + fprintf(stderr, "ERROR [RD:%d]: %u != 0\n", __LINE__, l); + exit(EXIT_FAILURE); + } + } + ck_brlock_read_unlock(&r); + } + + ck_brlock_read_unregister(&lock, &r); + return (NULL); +} + +int +main(int argc, char *argv[]) +{ + pthread_t *threads; + int i; + + if (argc != 3) { + fprintf(stderr, "Usage: validate \n"); + exit(EXIT_FAILURE); + } + + nthr = atoi(argv[1]); + if (nthr <= 0) { + fprintf(stderr, "ERROR: Number of threads must be greater than 0\n"); + exit(EXIT_FAILURE); + } + + threads = malloc(sizeof(pthread_t) * nthr); + if (threads == NULL) { + fprintf(stderr, "ERROR: Could not allocate thread structures\n"); + exit(EXIT_FAILURE); + } + + a.delta = atoi(argv[2]); + + fprintf(stderr, "Creating threads (mutual exclusion)..."); + for (i = 0; i < nthr; i++) { + if (pthread_create(&threads[i], NULL, thread, NULL)) { + fprintf(stderr, "ERROR: Could not create thread %d\n", i); + exit(EXIT_FAILURE); + } + } + fprintf(stderr, "done\n"); + + fprintf(stderr, "Waiting for threads to finish correctness regression..."); + for (i = 0; i < nthr; i++) + pthread_join(threads[i], NULL); + fprintf(stderr, "done (passed)\n"); + + return (0); +} +