1 files changed, 59 insertions, 107 deletions
diff --git a/ruby_atomic.h b/ruby_atomic.h
index 4a53e60824..cbcfe682ce 100644
--- a/ruby_atomic.h
+++ b/ruby_atomic.h
@@ -1,121 +1,73 @@
-#ifndef RUBY_ATOMIC_H
-#define RUBY_ATOMIC_H
+#ifndef INTERNAL_ATOMIC_H
+#define INTERNAL_ATOMIC_H
 
-#if 0
-#elif defined HAVE_GCC_ATOMIC_BUILTINS
-/* @shyouhei hack to support atomic operations in case of gcc. Gcc
- * has its own pseudo-insns to support them.  See info, or
- * http://gcc.gnu.org/onlinedocs/gcc/Atomic-Builtins.html */
+#include "ruby/atomic.h"
+#ifdef HAVE_STDATOMIC_H
+# include <stdatomic.h>
+#endif
 
-typedef unsigned int rb_atomic_t; /* Anything OK */
-# define ATOMIC_SET(var, val)  (void)__sync_lock_test_and_set(&(var), (val))
-# define ATOMIC_INC(var) __sync_fetch_and_add(&(var), 1)
-# define ATOMIC_DEC(var) __sync_fetch_and_sub(&(var), 1)
-# define ATOMIC_OR(var, val) __sync_or_and_fetch(&(var), (val))
-# define ATOMIC_EXCHANGE(var, val) __sync_lock_test_and_set(&(var), (val))
-# define ATOMIC_CAS(var, oldval, newval) __sync_val_compare_and_swap(&(var), (oldval), (newval))
+#define RUBY_ATOMIC_VALUE_LOAD(x) rbimpl_atomic_value_load(&(x), RBIMPL_ATOMIC_SEQ_CST)
 
-# define ATOMIC_SIZE_ADD(var, val) __sync_fetch_and_add(&(var), (val))
-# define ATOMIC_SIZE_SUB(var, val) __sync_fetch_and_sub(&(var), (val))
-# define ATOMIC_SIZE_INC(var) __sync_fetch_and_add(&(var), 1)
-# define ATOMIC_SIZE_DEC(var) __sync_fetch_and_sub(&(var), 1)
-# define ATOMIC_SIZE_EXCHANGE(var, val) __sync_lock_test_and_set(&(var), (val))
+/* shim macros only */
+#define ATOMIC_ADD(var, val) RUBY_ATOMIC_ADD(var, val)
+#define ATOMIC_CAS(var, oldval, newval) RUBY_ATOMIC_CAS(var, oldval, newval)
+#define ATOMIC_DEC(var) RUBY_ATOMIC_DEC(var)
+#define ATOMIC_EXCHANGE(var, val) RUBY_ATOMIC_EXCHANGE(var, val)
+#define ATOMIC_FETCH_ADD(var, val) RUBY_ATOMIC_FETCH_ADD(var, val)
+#define ATOMIC_FETCH_SUB(var, val) RUBY_ATOMIC_FETCH_SUB(var, val)
+#define ATOMIC_INC(var) RUBY_ATOMIC_INC(var)
+#define ATOMIC_OR(var, val) RUBY_ATOMIC_OR(var, val)
+#define ATOMIC_PTR_CAS(var, oldval, newval) RUBY_ATOMIC_PTR_CAS(var, oldval, newval)
+#define ATOMIC_PTR_EXCHANGE(var, val) RUBY_ATOMIC_PTR_EXCHANGE(var, val)
+#define ATOMIC_SET(var, val) RUBY_ATOMIC_SET(var, val)
+#define ATOMIC_SIZE_ADD(var, val) RUBY_ATOMIC_SIZE_ADD(var, val)
+#define ATOMIC_SIZE_CAS(var, oldval, newval) RUBY_ATOMIC_SIZE_CAS(var, oldval, newval)
+#define ATOMIC_SIZE_DEC(var) RUBY_ATOMIC_SIZE_DEC(var)
+#define ATOMIC_SIZE_EXCHANGE(var, val) RUBY_ATOMIC_SIZE_EXCHANGE(var, val)
+#define ATOMIC_SIZE_INC(var) RUBY_ATOMIC_SIZE_INC(var)
+#define ATOMIC_SIZE_SUB(var, val) RUBY_ATOMIC_SIZE_SUB(var, val)
+#define ATOMIC_SUB(var, val) RUBY_ATOMIC_SUB(var, val)
+#define ATOMIC_VALUE_CAS(var, oldval, val) RUBY_ATOMIC_VALUE_CAS(var, oldval, val)
+#define ATOMIC_VALUE_EXCHANGE(var, val) RUBY_ATOMIC_VALUE_EXCHANGE(var, val)
 
-#elif defined _WIN32
-#if defined _MSC_VER && _MSC_VER > 1200
-#pragma intrinsic(_InterlockedOr)
-#endif
-typedef LONG rb_atomic_t;
+#define ATOMIC_LOAD_RELAXED(var) rbimpl_atomic_load(&(var), RBIMPL_ATOMIC_RELAXED)
 
-# define ATOMIC_SET(var, val) InterlockedExchange(&(var), (val))
-# define ATOMIC_INC(var) InterlockedIncrement(&(var))
-# define ATOMIC_DEC(var) InterlockedDecrement(&(var))
-#if defined __GNUC__
-# define ATOMIC_OR(var, val) __asm__("lock\n\t" "orl\t%1, %0" : "=m"(var) : "Ir"(val))
-#elif defined _MSC_VER && _MSC_VER <= 1200
-# define ATOMIC_OR(var, val) rb_w32_atomic_or(&(var), (val))
-static inline void
-rb_w32_atomic_or(volatile rb_atomic_t *var, rb_atomic_t val)
+typedef RBIMPL_ALIGNAS(8) uint64_t rbimpl_atomic_uint64_t;
+
+static inline uint64_t
+rbimpl_atomic_u64_load_relaxed(const volatile rbimpl_atomic_uint64_t *value)
 {
-#ifdef _M_IX86
-    __asm mov eax, var;
-    __asm mov ecx, val;
-    __asm lock or [eax], ecx;
+#if defined(HAVE_GCC_ATOMIC_BUILTINS_64)
+    return __atomic_load_n(value, __ATOMIC_RELAXED);
+#elif defined(_WIN32)
+    uint64_t val = *value;
+    return InterlockedCompareExchange64(RBIMPL_CAST((uint64_t *)value), val, val);
+#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
+    uint64_t val = *value;
+    return atomic_cas_64(value, val, val);
 #else
-#error unsupported architecture
+    // TODO: stdatomic
+
+    return *value;
 #endif
 }
-#else
-# define ATOMIC_OR(var, val) _InterlockedOr(&(var), (val))
-#endif
-# define ATOMIC_EXCHANGE(var, val) InterlockedExchange(&(var), (val))
-# define ATOMIC_CAS(var, oldval, newval) InterlockedCompareExchange(&(var), (newval), (oldval))
-# ifdef _M_AMD64
-#  define ATOMIC_SIZE_ADD(var, val) InterlockedExchangeAdd64(&(var), (val))
-#  define ATOMIC_SIZE_SUB(var, val) InterlockedExchangeAdd64(&(var), -(val))
-#  define ATOMIC_SIZE_INC(var) InterlockedIncrement64(&(var))
-#  define ATOMIC_SIZE_DEC(var) InterlockedDecrement64(&(var))
-#  define ATOMIC_SIZE_EXCHANGE(var, val) InterlockedExchange64(&(var), (val))
-# else
-#  define ATOMIC_SIZE_ADD(var, val) InterlockedExchangeAdd((LONG *)&(var), (val))
-#  define ATOMIC_SIZE_SUB(var, val) InterlockedExchangeAdd((LONG *)&(var), -(val))
-#  define ATOMIC_SIZE_INC(var) InterlockedIncrement((LONG *)&(var))
-#  define ATOMIC_SIZE_DEC(var) InterlockedDecrement((LONG *)&(var))
-#  define ATOMIC_SIZE_EXCHANGE(var, val) InterlockedExchange((LONG *)&(var), (val))
-# endif
-
-#elif defined(__sun) && defined(HAVE_ATOMIC_H)
-#include <atomic.h>
-typedef unsigned int rb_atomic_t;
-
-# define ATOMIC_SET(var, val) (void)atomic_swap_uint(&(var), (val))
-# define ATOMIC_INC(var) atomic_inc_uint(&(var))
-# define ATOMIC_DEC(var) atomic_dec_uint(&(var))
-# define ATOMIC_OR(var, val) atomic_or_uint(&(var), (val))
-# define ATOMIC_EXCHANGE(var, val) atomic_swap_uint(&(var), (val))
-# define ATOMIC_CAS(var, oldval, newval) atomic_cas_uint(&(var), (oldval), (newval))
-
-# if SIZEOF_SIZE_T == SIZEOF_LONG
-#  define ATOMIC_SIZE_ADD(var, val) atomic_add_long(&(var), (val))
-#  define ATOMIC_SIZE_SUB(var, val) atomic_add_long(&(var), -(val))
-#  define ATOMIC_SIZE_INC(var) atomic_inc_ulong(&(var))
-#  define ATOMIC_SIZE_DEC(var) atomic_dec_ulong(&(var))
-#  define ATOMIC_SIZE_EXCHANGE(var, val) atomic_swap_ulong(&(var), (val))
-# else
-#  define ATOMIC_SIZE_ADD(var, val) atomic_add_int(&(var), (val))
-#  define ATOMIC_SIZE_SUB(var, val) atomic_add_int(&(var), -(val))
-#  define ATOMIC_SIZE_INC(var) atomic_inc_uint(&(var))
-#  define ATOMIC_SIZE_DEC(var) atomic_dec_uint(&(var))
-#  define ATOMIC_SIZE_EXCHANGE(var, val) atomic_swap_uint(&(var), (val))
-# endif
+#define ATOMIC_U64_LOAD_RELAXED(var) rbimpl_atomic_u64_load_relaxed(&(var))
 
+static inline void
+rbimpl_atomic_u64_set_relaxed(volatile rbimpl_atomic_uint64_t *address, uint64_t value)
+{
+#if defined(HAVE_GCC_ATOMIC_BUILTINS_64)
+    __atomic_store_n(address, value, __ATOMIC_RELAXED);
+#elif defined(_WIN32)
+    InterlockedExchange64(address, value);
+#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
+    atomic_swap_64(address, value);
 #else
-typedef int rb_atomic_t;
-#define NEED_RUBY_ATOMIC_OPS
-extern rb_atomic_t ruby_atomic_exchange(rb_atomic_t *ptr, rb_atomic_t val);
-extern rb_atomic_t ruby_atomic_compare_and_swap(rb_atomic_t *ptr,
-						rb_atomic_t cmp,
-						rb_atomic_t newval);
-
-# define ATOMIC_SET(var, val) (void)((var) = (val))
-# define ATOMIC_INC(var) ((var)++)
-# define ATOMIC_DEC(var) ((var)--)
-# define ATOMIC_OR(var, val) ((var) |= (val))
-# define ATOMIC_EXCHANGE(var, val) ruby_atomic_exchange(&(var), (val))
-# define ATOMIC_CAS(var, oldval, newval) ruby_atomic_compare_and_swap(&(var), (oldval), (newval))
+    // TODO: stdatomic
 
-# define ATOMIC_SIZE_ADD(var, val) (void)((var) += (val))
-# define ATOMIC_SIZE_SUB(var, val) (void)((var) -= (val))
-# define ATOMIC_SIZE_INC(var) ((var)++)
-# define ATOMIC_SIZE_DEC(var) ((var)--)
-# define ATOMIC_SIZE_EXCHANGE(var, val) atomic_size_exchange(&(var), (val))
-static inline size_t
-atomic_size_exchange(size_t *ptr, size_t val)
-{
-    size_t old = *ptr;
-    *ptr = val;
-    return old;
-}
+    *address = value;
 #endif
+}
+#define ATOMIC_U64_SET_RELAXED(var, val) rbimpl_atomic_u64_set_relaxed(&(var), val)
 
-#endif /* RUBY_ATOMIC_H */
+#endif