Add thread-based RCU

linD026 · linD026 · commit b67ba09e42b8 · 2022-04-24T18:12:54.000+08:00
Add the Linux Kernel style thread-based simple RCU. It supports sparse checking [1]. With sparse, it will report: main.c: note: in included file: thrd_rcu.h:95:42: warning: Using plain integer as NULL pointer thrd_rcu.h:95:42: warning: Using plain integer as NULL pointer It is from the pthread_mutex_t initializer, we can ignore it. [1] https://www.kernel.org/doc/html/latest/dev-tools/sparse.html
diff --git a/thrd_rcu/Makefile b/thrd_rcu/Makefile
@@ -0,0 +1,19 @@
+CFLAGS = -Wall
+CFLAGS += -g
+CFLAGS += -D'READER_NUM=10'
+CFLAGS += -D'UPDATER_NUM=1'
+CFLAGS += -lpthread
+
+all:
+	$(CC) -o main main.c $(CFLAGS)
+
+# Semantic Checker
+# https://www.kernel.org/doc/html/latest/dev-tools/sparse.html
+sparse:
+	sparse main.c $(CFLAGS)
+
+indent:
+	clang-format -i *.[ch]
+
+clean:
+	rm -f main
diff --git a/thrd_rcu/main.c b/thrd_rcu/main.c
@@ -0,0 +1,75 @@
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "thrd_rcu.h"
+
+struct test {
+    unsigned int count;
+};
+
+static struct test __rcu *foo;
+
+static void *reader_side(void *argv)
+{
+    struct test __allow_unused *tmp;
+
+    if (rcu_init())
+        abort();
+
+    rcu_read_lock();
+
+    tmp = rcu_dereference(foo);
+
+    printf("[reader %u] %u\n", current_tid(), tmp->count);
+
+    rcu_read_unlock();
+
+    pthread_exit(NULL);
+}
+
+static void *updater_side(void *argv)
+{
+    struct test *oldp;
+    struct test *newval = (struct test *) malloc(sizeof(struct test));
+    newval->count = current_tid();
+
+    printf("[updater %u]\n", newval->count);
+
+    oldp = rcu_assign_pointer(foo, newval);
+
+    synchronize_rcu();
+    free(oldp);
+
+    pthread_exit(NULL);
+}
+
+int main(int argc, char *argv[])
+{
+    pthread_t reader[READER_NUM];
+    pthread_t updater[UPDATER_NUM];
+    int i;
+
+    foo = (struct test __rcu *) malloc(sizeof(struct test));
+    rcu_uncheck(foo)->count = 0;
+
+    for (i = 0; i < READER_NUM / 2; i++)
+        pthread_create(&reader[i], NULL, reader_side, NULL);
+
+    for (i = 0; i < UPDATER_NUM; i++)
+        pthread_create(&updater[i], NULL, updater_side, NULL);
+
+    for (i = READER_NUM / 2; i < READER_NUM; i++)
+        pthread_create(&reader[i], NULL, reader_side, NULL);
+
+    for (i = 0; i < READER_NUM; i++)
+        pthread_join(reader[i], NULL);
+
+    for (i = 0; i < UPDATER_NUM; i++)
+        pthread_join(updater[i], NULL);
+
+    free(rcu_uncheck(foo));
+    rcu_clean();
+
+    return 0;
+}
diff --git a/thrd_rcu/thrd_rcu.h b/thrd_rcu/thrd_rcu.h
@@ -0,0 +1,215 @@
+#ifndef __THRD_RCU_H__
+#define __THRD_RCU_H__
+
+/* lock primitives from pthread and compiler primitives */
+
+#include <pthread.h>
+
+typedef pthread_mutex_t spinlock_t;
+
+#define SPINLOCK_INIT PTHREAD_MUTEX_INITIALIZER
+
+static __inline__ void spin_lock(spinlock_t *sp)
+{
+    int ret;
+
+    ret = pthread_mutex_lock(sp);
+    if (ret != 0) {
+        fprintf(stderr, "spin_lock:pthread_mutex_lock %d\n", ret);
+        abort();
+    }
+}
+
+static __inline__ void spin_unlock(spinlock_t *sp)
+{
+    int ret;
+
+    ret = pthread_mutex_unlock(sp);
+    if (ret != 0) {
+        fprintf(stderr, "spin_unlock:pthread_mutex_unlock %d\n", ret);
+        abort();
+    }
+}
+
+#define current_tid() (unsigned int) pthread_self()
+
+#define ACCESS_ONCE(x) (*(volatile __typeof__(x) *) &(x))
+#define READ_ONCE(x)                         \
+    ({                                       \
+        __typeof__(x) ___x = ACCESS_ONCE(x); \
+        ___x;                                \
+    })
+#define WRITE_ONCE(x, val)      \
+    do {                        \
+        ACCESS_ONCE(x) = (val); \
+    } while (0)
+#define barrier() __asm__ __volatile__("" : : : "memory")
+#define __allow_unused __attribute__((unused))
+#define smp_mb() __asm__ __volatile__("mfence" : : : "memory")
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#ifdef __CHECKER__
+#define __rcu __attribute__((noderef, address_space(__rcu)))
+#define rcu_check_sparse(p, space) ((void) (((typeof(*p) space *) p) == p))
+#define __force __attribute__((force))
+#define rcu_uncheck(p) ((__typeof__(*p) __force *) p)
+#define rcu_check(p) ((__typeof__(*p) __force __rcu *) p)
+#else
+#define __rcu
+#define rcu_check_sparse(p, space)
+#define __force
+#define rcu_uncheck(p) p
+#define rcu_check(p) p
+#endif /* __CHECKER__ */
+
+/* Avoid false sharing */
+#define __rcu_aligned __attribute__((aligned(128)))
+
+struct rcu_node {
+    unsigned int tid;
+    int rcu_nesting[2];
+    struct rcu_node *next;
+} __rcu_aligned;
+
+struct rcu_data {
+    unsigned int nr_thread;
+    struct rcu_node *head;
+    unsigned int rcu_thrd_nesting_idx;
+    spinlock_t sp;
+};
+
+/* Easy to use */
+#define __rcu_thrd_idx rcu_data.rcu_thrd_nesting_idx
+#define __rcu_thrd_nesting(ptr) \
+    ptr->rcu_nesting[READ_ONCE(__rcu_thrd_idx) & 0x01]
+#define rcu_thrd_nesting __rcu_thrd_nesting(__rcu_per_thrd_ptr)
+
+static struct rcu_data rcu_data = {.nr_thread = 0,
+                                   .head = NULL,
+                                   .rcu_thrd_nesting_idx = 0,
+                                   .sp = SPINLOCK_INIT};
+static __thread struct rcu_node *__rcu_per_thrd_ptr;
+
+static __inline__ struct rcu_node *__rcu_node_add(unsigned int tid)
+{
+    struct rcu_node **indirect = &rcu_data.head;
+    struct rcu_node *node;
+
+    node = (struct rcu_node *) malloc(sizeof(struct rcu_node));
+    if (!node) {
+        fprintf(stderr, "__rcu_node_add: malloc failed\n");
+        abort();
+    }
+
+    node->tid = tid;
+    node->rcu_nesting[0] = 0;
+    node->rcu_nesting[1] = 0;
+    node->next = NULL;
+
+    spin_lock(&rcu_data.sp);
+
+    while (*indirect) {
+        if ((*indirect)->tid == node->tid) {
+            spin_unlock(&rcu_data.sp);
+            free(node);
+            return NULL;
+        }
+        indirect = &(*indirect)->next;
+    }
+
+    *indirect = node;
+    rcu_data.nr_thread++;
+
+    spin_unlock(&rcu_data.sp);
+
+    smp_mb();
+
+    return node;
+}
+
+static __inline__ int rcu_init(void)
+{
+    unsigned int tid = current_tid();
+
+    __rcu_per_thrd_ptr = __rcu_node_add(tid);
+
+    return (__rcu_per_thrd_ptr == NULL) ? -ENOMEM : 0;
+}
+
+static __inline__ void rcu_clean(void)
+{
+    struct rcu_node *node, *tmp;
+
+    spin_lock(&rcu_data.sp);
+
+    for (node = rcu_data.head; node != NULL; node = tmp) {
+        tmp = node->next;
+        free(node);
+    }
+
+    rcu_data.head = NULL;
+    rcu_data.nr_thread = 0;
+
+    spin_unlock(&rcu_data.sp);
+}
+
+/* The per-thread reference count will only modified by their owner
+ * thread but will read by other threads. So here we use WRITE_ONCE().
+ */
+static __inline__ void rcu_read_lock(void)
+{
+    WRITE_ONCE(rcu_thrd_nesting, 1);
+}
+
+static __inline__ void rcu_read_unlock(void)
+{
+    WRITE_ONCE(rcu_thrd_nesting, 0);
+}
+
+static __inline__ void synchronize_rcu(void)
+{
+    struct rcu_node *node;
+
+    smp_mb();
+
+    spin_lock(&rcu_data.sp);
+
+    /* When the rcu_thrd_nesting is odd, which means that the LSB set 1,
+     * that thread is in the read-side critical section. Also, we need
+     * to skip the read side when it is in the new grace period.
+     */
+    for (node = rcu_data.head; node != NULL; node = node->next) {
+        while (READ_ONCE(__rcu_thrd_nesting(node)) & 0x1) {
+            barrier();
+        }
+    }
+
+    /* Going to next grace period */
+    __atomic_fetch_add(&__rcu_thrd_idx, 1, __ATOMIC_RELEASE);
+
+    spin_unlock(&rcu_data.sp);
+
+    smp_mb();
+}
+
+#define rcu_dereference(p)                                                 \
+    ({                                                                     \
+        __typeof__(*p) *__r_d_p = (__typeof__(*p) __force *) READ_ONCE(p); \
+        rcu_check_sparse(p, __rcu);                                        \
+        __r_d_p;                                                           \
+    })
+
+#define rcu_assign_pointer(p, v)                                               \
+    ({                                                                         \
+        __typeof__(*p) *__r_a_p =                                              \
+            (__typeof__(*p) __force *) __atomic_exchange_n(                    \
+                &(p), (__typeof__(*(p)) __force __rcu *) v, __ATOMIC_RELEASE); \
+        rcu_check_sparse(p, __rcu);                                            \
+        __r_a_p;                                                               \
+    })
+
+#endif /* __THRD_RCU_H__ */
