Bug #25952493: MOVE LOCK-FREE DATA STRUCTURES TO STD::ATOMIC

Move all of our lock-free data structures std::atomic. This gives greatly increased type safety, as well as being one part on the step towards removing our own my_atomic system. Change-Id: Ic8dccd14cd85caf6fdb787d52335668188895644
polardb · Apr 26, 2017 · 78c6c70 · 78c6c70
1 parent 044c04e
commit 78c6c70
Show file tree

Hide file tree

Showing 6 changed files with 146 additions and 109 deletions.
diff --git a/include/lf.h b/include/lf.h
@@ -25,8 +25,9 @@
 #include <stddef.h>
 #include <sys/types.h>
 
+#include <atomic>
+
 #include "hash.h"
-#include "my_atomic.h"
 #include "my_inttypes.h"
 #include "my_macros.h"
 #include "mysql/psi/mysql_statement.h"
@@ -45,7 +46,7 @@ C_MODE_START
 #define LF_DYNARRAY_LEVELS       4
 
 typedef struct {
-  void * volatile level[LF_DYNARRAY_LEVELS];
+  std::atomic<void *> level[LF_DYNARRAY_LEVELS];
   uint size_of_element;
 } LF_DYNARRAY;
 
@@ -71,16 +72,16 @@ typedef struct {
   lf_pinbox_free_func *free_func;
   void *free_func_arg;
   uint free_ptr_offset;
-  uint32 volatile pinstack_top_ver;         /* this is a versioned pointer */
-  uint32 volatile pins_in_array;            /* number of elements in array */
+  std::atomic<uint32> pinstack_top_ver;    /* this is a versioned pointer */
+  std::atomic<uint32> pins_in_array;       /* number of elements in array */
 } LF_PINBOX;
 
 typedef struct st_lf_pins {
-  void * volatile pin[LF_PINBOX_PINS];
+  std::atomic<void *> pin[LF_PINBOX_PINS];
   LF_PINBOX *pinbox;
   void  *purgatory;
   uint32 purgatory_count;
-  uint32 volatile link;
+  std::atomic<uint32> link;
 /* we want sizeof(LF_PINS) to be 64 to avoid false sharing */
 #if SIZEOF_INT*2+SIZEOF_CHARP*(LF_PINBOX_PINS+2) != 64
   char pad[64-sizeof(uint32)*2-sizeof(void*)*(LF_PINBOX_PINS+2)];
@@ -106,15 +107,15 @@ static inline void lf_pin(LF_PINS *pins, int pin, void *addr)
 #if defined(__GNUC__) && defined(MY_LF_EXTRA_DEBUG)
   assert(pin < LF_NUM_PINS_IN_THIS_FILE);
 #endif
-  my_atomic_storeptr(&pins->pin[pin], addr);
+  pins->pin[pin].store(addr);
 }
 
 static inline void lf_unpin(LF_PINS *pins, int pin)
 {
 #if defined(__GNUC__) && defined(MY_LF_EXTRA_DEBUG)
   assert(pin < LF_NUM_PINS_IN_THIS_FILE);
 #endif
-  my_atomic_storeptr(&pins->pin[pin], NULL);
+  pins->pin[pin].store(nullptr);
 }
 
 void lf_pinbox_init(LF_PINBOX *pinbox, uint free_ptr_offset,
@@ -132,9 +133,9 @@ typedef void lf_allocator_func(uchar *);
 
 typedef struct st_lf_allocator {
   LF_PINBOX pinbox;
-  uchar * volatile top;
+  std::atomic<uchar *> top;
   uint element_size;
-  uint32 volatile mallocs;
+  std::atomic<uint32> mallocs;
   lf_allocator_func *constructor; /* called, when an object is malloc()'ed */
   lf_allocator_func *destructor;  /* called, when an object is free()'d    */
 } LF_ALLOCATOR;
@@ -174,8 +175,8 @@ typedef struct st_lf_hash {
   uint key_offset, key_length;          /* see HASH */
   uint element_size;                    /* size of memcpy'ed area on insert */
   uint flags;                           /* LF_HASH_UNIQUE, etc */
-  int32 volatile size;                  /* size of array */
-  int32 volatile count;                 /* number of elements in the hash */
+  std::atomic<int32> size;              /* size of array */
+  std::atomic<int32> count;             /* number of elements in the hash */
   /**
     "Initialize" hook - called to finish initialization of object provided by
      LF_ALLOCATOR (which is pointed by "dst" parameter) and set element key

diff --git a/mysys/lf_alloc-pin.cc b/mysys/lf_alloc-pin.cc
@@ -1,5 +1,5 @@
 /* QQ: TODO multi-pinbox */
-/* Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2006, 2017, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -17,6 +17,13 @@
 #include <stddef.h>
 #include <sys/types.h>
 
+#include <atomic>
+
+static_assert(
+  sizeof(std::atomic<void *>) == sizeof(void *),
+  "We happily cast to and from std::atomic<void *>, so they need to be at least"
+  "nominally compatible.");
+
 /**
   @file mysys/lf_alloc-pin.cc
   wait-free concurrent allocator based on pinning addresses.
@@ -164,7 +171,7 @@ LF_PINS *lf_pinbox_get_pins(LF_PINBOX *pinbox)
     if (!(pins= top_ver % LF_PINBOX_MAX_PINS))
     {
       /* the stack of free elements is empty */
-      pins= my_atomic_add32((int32 volatile*) &pinbox->pins_in_array, 1)+1;
+      pins= pinbox->pins_in_array.fetch_add(1) + 1;
       if (unlikely(pins >= LF_PINBOX_MAX_PINS))
         return 0;
       /*
@@ -178,9 +185,10 @@ LF_PINS *lf_pinbox_get_pins(LF_PINBOX *pinbox)
     }
     el= (LF_PINS *)lf_dynarray_value(&pinbox->pinarray, pins);
     next= el->link;
-  } while (!my_atomic_cas32((int32 volatile*) &pinbox->pinstack_top_ver,
-                            (int32*) &top_ver,
-                            top_ver-pins+next+LF_PINBOX_MAX_PINS));
+  } while (!atomic_compare_exchange_strong(
+             &pinbox->pinstack_top_ver,
+             &top_ver,
+             top_ver-pins+next+LF_PINBOX_MAX_PINS));
   /*
     set el->link to the index of el in the dynarray (el->link has two usages:
     - if element is allocated, it's its own index
@@ -232,9 +240,10 @@ void lf_pinbox_put_pins(LF_PINS *pins)
   do
   {
     pins->link= top_ver % LF_PINBOX_MAX_PINS;
-  } while (!my_atomic_cas32((int32 volatile*) &pinbox->pinstack_top_ver,
-                            (int32*) &top_ver,
-                            top_ver-pins->link+nr+LF_PINBOX_MAX_PINS));
+  } while (!atomic_compare_exchange_strong(
+             &pinbox->pinstack_top_ver,
+             &top_ver,
+             top_ver-pins->link+nr+LF_PINBOX_MAX_PINS));
 }
 
 /*
@@ -339,7 +348,12 @@ static void lf_pinbox_real_free(LF_PINS *pins)
   }
 }
 
-#define next_node(P, X) (*((uchar * volatile *)(((uchar *)(X)) + (P)->free_ptr_offset)))
+static inline std::atomic<uchar *>& next_node(LF_PINBOX *P, uchar *X)
+{
+  std::atomic<uchar *> *free_ptr= (std::atomic<uchar *> *)(X + P->free_ptr_offset);
+  return *free_ptr;
+}
+
 #define anext_node(X) next_node(&allocator->pinbox, (X))
 
 /* lock-free memory allocator for fixed-size objects */
@@ -355,20 +369,15 @@ LF_REQUIRE_PINS(1)
     first->el->el->....->el->last. Use first==last to free only one element.
 */
 static void alloc_free(uchar *first,
-                       uchar volatile *last,
+                       uchar *last,
                        LF_ALLOCATOR *allocator)
 {
-  /*
-    we need a union here to access type-punned pointer reliably.
-    otherwise gcc -fstrict-aliasing will not see 'tmp' changed in the loop
-  */
-  union { uchar * node; void *ptr; } tmp;
-  tmp.node= allocator->top;
+  uchar *node= allocator->top;
   do
   {
-    anext_node(last)= tmp.node;
-  } while (!my_atomic_casptr((void **)(char *)&allocator->top,
-                             (void **)&tmp.ptr, first) && LF_BACKOFF);
+    anext_node(last)= node;
+  } while (!atomic_compare_exchange_strong(
+             &allocator->top, &node, first) && LF_BACKOFF);
 }
 
 /**
@@ -453,12 +462,12 @@ void *lf_alloc_new(LF_PINS *pins)
         allocator->constructor(node);
 #ifdef MY_LF_EXTRA_DEBUG
       if (likely(node != 0))
-        my_atomic_add32(&allocator->mallocs, 1);
+        ++allocator->mallocs;
 #endif
       break;
     }
-    if (my_atomic_casptr((void **)(char *)&allocator->top,
-                         reinterpret_cast<void **>(&node), anext_node(node)))
+    if (atomic_compare_exchange_strong(
+          &allocator->top, &node, anext_node(node).load()))
       break;
   }
   lf_unpin(pins, 0);

diff --git a/mysys/lf_dynarray.cc b/mysys/lf_dynarray.cc
@@ -1,4 +1,4 @@
-/* Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2006, 2017, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -40,7 +40,6 @@
 #include <sys/types.h>
 
 #include "lf.h"
-#include "my_atomic.h"
 #include "my_compiler.h"
 #include "my_inttypes.h"
 #include "my_macros.h"
@@ -54,7 +53,7 @@ void lf_dynarray_init(LF_DYNARRAY *array, uint element_size)
   array->size_of_element= element_size;
 }
 
-static void recursive_free(void **alloc, int level)
+static void recursive_free(std::atomic<void *> *alloc, int level)
 {
   if (!alloc)
     return;
@@ -63,7 +62,7 @@ static void recursive_free(void **alloc, int level)
   {
     int i;
     for (i= 0; i < LF_DYNARRAY_LEVEL_LENGTH; i++)
-      recursive_free(static_cast<void**>(alloc[i]), level-1);
+      recursive_free(static_cast<std::atomic<void *> *>(alloc[i].load()), level-1);
     my_free(alloc);
   }
   else
@@ -74,7 +73,7 @@ void lf_dynarray_destroy(LF_DYNARRAY *array)
 {
   int i;
   for (i= 0; i < LF_DYNARRAY_LEVELS; i++)
-    recursive_free(static_cast<void**>(array->level[i]), i);
+    recursive_free(static_cast<std::atomic<void *> *>(array->level[i].load()), i);
 }
 
 static const ulong dynarray_idxes_in_prev_levels[LF_DYNARRAY_LEVELS]=
@@ -103,12 +102,12 @@ static const ulong dynarray_idxes_in_prev_level[LF_DYNARRAY_LEVELS]=
 */
 void *lf_dynarray_lvalue(LF_DYNARRAY *array, uint idx)
 {
-  void * ptr, * volatile * ptr_ptr= 0;
+  void * ptr;
   int i;
 
   for (i= LF_DYNARRAY_LEVELS-1; idx < dynarray_idxes_in_prev_levels[i]; i--)
     /* no-op */;
-  ptr_ptr= &array->level[i];
+  std::atomic<void *> *ptr_ptr= &array->level[i];
   idx-= dynarray_idxes_in_prev_levels[i];
   for (; i > 0; i--)
   {
@@ -119,12 +118,12 @@ void *lf_dynarray_lvalue(LF_DYNARRAY *array, uint idx)
                              MYF(MY_WME|MY_ZEROFILL));
       if (unlikely(!alloc))
         return(NULL);
-      if (my_atomic_casptr(ptr_ptr, &ptr, alloc))
+      if (atomic_compare_exchange_strong(ptr_ptr, &ptr, alloc))
         ptr= alloc;
       else
         my_free(alloc);
     }
-    ptr_ptr= ((void **)ptr) + idx / dynarray_idxes_in_prev_level[i];
+    ptr_ptr= ((std::atomic<void *> *)ptr) + idx / dynarray_idxes_in_prev_level[i];
     idx%= dynarray_idxes_in_prev_level[i];
   }
   if (!(ptr= *ptr_ptr))
@@ -146,7 +145,7 @@ void *lf_dynarray_lvalue(LF_DYNARRAY *array, uint idx)
         data+= array->size_of_element - mod;
     }
     ((void **)data)[-1]= alloc; /* free() will need the original pointer */
-    if (my_atomic_casptr(ptr_ptr, &ptr, data))
+    if (atomic_compare_exchange_strong(ptr_ptr, &ptr, static_cast<void *>(data)))
       ptr= data;
     else
       my_free(alloc);
@@ -160,18 +159,18 @@ void *lf_dynarray_lvalue(LF_DYNARRAY *array, uint idx)
 */
 void *lf_dynarray_value(LF_DYNARRAY *array, uint idx)
 {
-  void * ptr, * volatile * ptr_ptr= 0;
+  void * ptr;
   int i;
 
   for (i= LF_DYNARRAY_LEVELS-1; idx < dynarray_idxes_in_prev_levels[i]; i--)
     /* no-op */;
-  ptr_ptr= &array->level[i];
+  std::atomic<void *> *ptr_ptr= &array->level[i];
   idx-= dynarray_idxes_in_prev_levels[i];
   for (; i > 0; i--)
   {
     if (!(ptr= *ptr_ptr))
       return(NULL);
-    ptr_ptr= ((void **)ptr) + idx / dynarray_idxes_in_prev_level[i];
+    ptr_ptr= ((std::atomic<void *> *)ptr) + idx / dynarray_idxes_in_prev_level[i];
     idx %= dynarray_idxes_in_prev_level[i];
   }
   if (!(ptr= *ptr_ptr))