diff options
Diffstat (limited to 'include/ruby/atomic.h')
| -rw-r--r-- | include/ruby/atomic.h | 1037 |
1 files changed, 166 insertions, 871 deletions
diff --git a/include/ruby/atomic.h b/include/ruby/atomic.h index eb106631f6..a898c30ddf 100644 --- a/include/ruby/atomic.h +++ b/include/ruby/atomic.h @@ -1,941 +1,236 @@ -#ifndef RUBY_ATOMIC_H /*-*-C++-*-vi:se ft=cpp:*/ +#ifndef RUBY_ATOMIC_H #define RUBY_ATOMIC_H -/** - * @file - * @author Ruby developers <ruby-core@ruby-lang.org> - * @copyright This file is a part of the programming language Ruby. - * Permission is hereby granted, to either redistribute and/or - * modify this file, provided that the conditions mentioned in the - * file COPYING are met. Consult the file for details. - * @warning Symbols prefixed with either `RBIMPL` or `rbimpl` are - * implementation details. Don't take them as canon. They could - * rapidly appear then vanish. The name (path) of this header file - * is also an implementation detail. Do not expect it to persist - * at the place it is now. Developers are free to move it anywhere - * anytime at will. - * @note To ruby-core: remember that this header can be possibly - * recursively included from extension libraries written in C++. - * Do not expect for instance `__VA_ARGS__` is always available. - * We assume C99 for ruby itself but we don't assume languages of - * extension libraries. They could be written in C++98. - * @brief Atomic operations - * - * Basically, if we could assume either C11 or C++11, these macros are just - * redundant. Sadly we cannot. We have to do them ourselves. - */ - -#include "ruby/internal/config.h" - -#ifdef STDC_HEADERS -# include <stddef.h> /* size_t */ -#endif - -#ifdef HAVE_SYS_TYPES_H -# include <sys/types.h> /* ssize_t */ -#endif - -#if RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0) -# pragma intrinsic(_InterlockedOr) -#elif defined(__sun) && defined(HAVE_ATOMIC_H) -# include <atomic.h> -#endif - -#include "ruby/assert.h" -#include "ruby/backward/2/limits.h" -#include "ruby/internal/attr/artificial.h" -#include "ruby/internal/attr/noalias.h" -#include "ruby/internal/attr/nonnull.h" -#include "ruby/internal/compiler_since.h" -#include "ruby/internal/cast.h" -#include "ruby/internal/value.h" -#include "ruby/internal/static_assert.h" -#include "ruby/internal/stdbool.h" /* - * Asserts that your environment supports more than one atomic types. These - * days systems tend to have such property (C11 was a standard of decades ago, - * right?) but we still support older ones. - */ -#if defined(__DOXYGEN__) || defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS) -# define RUBY_ATOMIC_GENERIC_MACRO 1 -#endif - -/** - * Type that is eligible for atomic operations. Depending on your host - * platform you might have more than one such type, but we choose one of them - * anyways. - */ -#if defined(__DOXYGEN__) -using rb_atomic_t = std::atomic<unsigned>; -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) -typedef unsigned int rb_atomic_t; -#elif defined(HAVE_GCC_SYNC_BUILTINS) -typedef unsigned int rb_atomic_t; -#elif defined(_WIN32) -typedef LONG rb_atomic_t; -#elif defined(__sun) && defined(HAVE_ATOMIC_H) -typedef unsigned int rb_atomic_t; -#else -# error No atomic operation found -#endif - -/** - * Atomically replaces the value pointed by `var` with the result of addition - * of `val` to the old value of `var`. - * - * @param var A variable of ::rb_atomic_t. - * @param val Value to add. - * @return What was stored in `var` before the addition. - * @post `var` holds `var + val`. - */ -#define RUBY_ATOMIC_FETCH_ADD(var, val) rbimpl_atomic_fetch_add(&(var), (val)) - -/** - * Atomically replaces the value pointed by `var` with the result of - * subtraction of `val` to the old value of `var`. - * - * @param var A variable of ::rb_atomic_t. - * @param val Value to subtract. - * @return What was stored in `var` before the subtraction. - * @post `var` holds `var - val`. - */ -#define RUBY_ATOMIC_FETCH_SUB(var, val) rbimpl_atomic_fetch_sub(&(var), (val)) - -/** - * Atomically replaces the value pointed by `var` with the result of - * bitwise OR between `val` and the old value of `var`. - * - * @param var A variable of ::rb_atomic_t. - * @param val Value to mix. - * @return void - * @post `var` holds `var | val`. - * @note For portability, this macro can return void. - */ -#define RUBY_ATOMIC_OR(var, val) rbimpl_atomic_or(&(var), (val)) - -/** - * Atomically replaces the value pointed by `var` with `val`. This is just an - * assignment, but you can additionally know the previous value. - * - * @param var A variable of ::rb_atomic_t. - * @param val Value to set. - * @return What was stored in `var` before the assignment. - * @post `var` holds `val`. - */ -#define RUBY_ATOMIC_EXCHANGE(var, val) rbimpl_atomic_exchange(&(var), (val)) - -/** - * Atomic compare-and-swap. This stores `val` to `var` if and only if the - * assignment changes the value of `var` from `oldval` to `newval`. You can - * detect whether the assignment happened or not using the return value. - * - * @param var A variable of ::rb_atomic_t. - * @param oldval Expected value of `var` before the assignment. - * @param newval What you want to store at `var`. - * @retval oldval Successful assignment (`var` is now `newval`). - * @retval otherwise Something else is at `var`; not updated. - */ -#define RUBY_ATOMIC_CAS(var, oldval, newval) \ - rbimpl_atomic_cas(&(var), (oldval), (newval)) - -/** - * Atomic load. This loads `var` with an atomic intrinsic and returns - * its value. - * - * @param var A variable of ::rb_atomic_t - * @return What was stored in `var`j - */ -#define RUBY_ATOMIC_LOAD(var) rbimpl_atomic_load(&(var)) - -/** - * Identical to #RUBY_ATOMIC_EXCHANGE, except for the return type. - * - * @param var A variable of ::rb_atomic_t. - * @param val Value to set. - * @return void - * @post `var` holds `val`. - */ -#define RUBY_ATOMIC_SET(var, val) rbimpl_atomic_set(&(var), (val)) - -/** - * Identical to #RUBY_ATOMIC_FETCH_ADD, except for the return type. - * - * @param var A variable of ::rb_atomic_t. - * @param val Value to add. - * @return void - * @post `var` holds `var + val`. - */ -#define RUBY_ATOMIC_ADD(var, val) rbimpl_atomic_add(&(var), (val)) - -/** - * Identical to #RUBY_ATOMIC_FETCH_SUB, except for the return type. - * - * @param var A variable of ::rb_atomic_t. - * @param val Value to subtract. - * @return void - * @post `var` holds `var - val`. + * - RUBY_ATOMIC_CAS, RUBY_ATOMIC_EXCHANGE, RUBY_ATOMIC_FETCH_*: + * return the old * value. + * - RUBY_ATOMIC_ADD, RUBY_ATOMIC_SUB, RUBY_ATOMIC_INC, RUBY_ATOMIC_DEC, RUBY_ATOMIC_OR, RUBY_ATOMIC_SET: + * may be void. */ -#define RUBY_ATOMIC_SUB(var, val) rbimpl_atomic_sub(&(var), (val)) - -/** - * Atomically increments the value pointed by `var`. - * - * @param var A variable of ::rb_atomic_t. - * @return void - * @post `var` holds `var + 1`. - */ -#define RUBY_ATOMIC_INC(var) rbimpl_atomic_inc(&(var)) - -/** - * Atomically decrements the value pointed by `var`. - * - * @param var A variable of ::rb_atomic_t. - * @return void - * @post `var` holds `var - 1`. - */ -#define RUBY_ATOMIC_DEC(var) rbimpl_atomic_dec(&(var)) - -/** - * Identical to #RUBY_ATOMIC_INC, except it expects its argument is `size_t`. - * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit. This - * should be used for size related operations to support such platforms. - * - * @param var A variable of `size_t`. - * @return void - * @post `var` holds `var + 1`. - */ -#define RUBY_ATOMIC_SIZE_INC(var) rbimpl_atomic_size_inc(&(var)) - -/** - * Identical to #RUBY_ATOMIC_DEC, except it expects its argument is `size_t`. - * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit. This - * should be used for size related operations to support such platforms. - * - * @param var A variable of `size_t`. - * @return void - * @post `var` holds `var - 1`. - */ -#define RUBY_ATOMIC_SIZE_DEC(var) rbimpl_atomic_size_dec(&(var)) - -/** - * Identical to #RUBY_ATOMIC_EXCHANGE, except it expects its arguments are - * `size_t`. There are cases where ::rb_atomic_t is 32bit while `size_t` is - * 64bit. This should be used for size related operations to support such - * platforms. - * - * @param var A variable of `size_t`. - * @param val Value to set. - * @return What was stored in `var` before the assignment. - * @post `var` holds `val`. - */ -#define RUBY_ATOMIC_SIZE_EXCHANGE(var, val) \ - rbimpl_atomic_size_exchange(&(var), (val)) - -/** - * Identical to #RUBY_ATOMIC_CAS, except it expects its arguments are `size_t`. - * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit. This - * should be used for size related operations to support such platforms. - * - * @param var A variable of `size_t`. - * @param oldval Expected value of `var` before the assignment. - * @param newval What you want to store at `var`. - * @retval oldval Successful assignment (`var` is now `newval`). - * @retval otherwise Something else is at `var`; not updated. - */ -#define RUBY_ATOMIC_SIZE_CAS(var, oldval, newval) \ - rbimpl_atomic_size_cas(&(var), (oldval), (newval)) - -/** - * Identical to #RUBY_ATOMIC_ADD, except it expects its arguments are `size_t`. - * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit. This - * should be used for size related operations to support such platforms. - * - * @param var A variable of `size_t`. - * @param val Value to add. - * @return void - * @post `var` holds `var + val`. - */ -#define RUBY_ATOMIC_SIZE_ADD(var, val) rbimpl_atomic_size_add(&(var), (val)) - -/** - * Identical to #RUBY_ATOMIC_SUB, except it expects its arguments are `size_t`. - * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit. This - * should be used for size related operations to support such platforms. - * - * @param var A variable of `size_t`. - * @param val Value to subtract. - * @return void - * @post `var` holds `var - val`. - */ -#define RUBY_ATOMIC_SIZE_SUB(var, val) rbimpl_atomic_size_sub(&(var), (val)) - -/** - * Identical to #RUBY_ATOMIC_EXCHANGE, except it expects its arguments are - * `void*`. There are cases where ::rb_atomic_t is 32bit while `void*` is - * 64bit. This should be used for pointer related operations to support such - * platforms. - * - * @param var A variable of `void *`. - * @param val Value to set. - * @return What was stored in `var` before the assignment. - * @post `var` holds `val`. - * - * @internal - * - * :FIXME: this `(void*)` cast is evil! However `void*` is incompatible with - * some pointers, most notably function pointers. - */ -#define RUBY_ATOMIC_PTR_EXCHANGE(var, val) \ - RBIMPL_CAST(rbimpl_atomic_ptr_exchange((void **)&(var), (void *)val)) - -/** - * Identical to #RUBY_ATOMIC_LOAD, except it expects its arguments are `void*`. - * There are cases where ::rb_atomic_t is 32bit while `void*` is 64bit. This - * should be used for size related operations to support such platforms. - * - * @param var A variable of `void*` - * @return The value of `var` (without tearing) - */ -#define RUBY_ATOMIC_PTR_LOAD(var) \ - RBIMPL_CAST(rbimpl_atomic_ptr_load((void **)&var)) - -/** - * Identical to #RUBY_ATOMIC_CAS, except it expects its arguments are `void*`. - * There are cases where ::rb_atomic_t is 32bit while `void*` is 64bit. This - * should be used for size related operations to support such platforms. - * - * @param var A variable of `void*`. - * @param oldval Expected value of `var` before the assignment. - * @param newval What you want to store at `var`. - * @retval oldval Successful assignment (`var` is now `newval`). - * @retval otherwise Something else is at `var`; not updated. - */ -#define RUBY_ATOMIC_PTR_CAS(var, oldval, newval) \ - RBIMPL_CAST(rbimpl_atomic_ptr_cas((void **)&(var), (oldval), (newval))) - -/** - * Identical to #RUBY_ATOMIC_EXCHANGE, except it expects its arguments are - * ::VALUE. There are cases where ::rb_atomic_t is 32bit while ::VALUE is - * 64bit. This should be used for pointer related operations to support such - * platforms. - * - * @param var A variable of ::VALUE. - * @param val Value to set. - * @return What was stored in `var` before the assignment. - * @post `var` holds `val`. - */ -#define RUBY_ATOMIC_VALUE_EXCHANGE(var, val) \ - rbimpl_atomic_value_exchange(&(var), (val)) - -/** - * Identical to #RUBY_ATOMIC_CAS, except it expects its arguments are ::VALUE. - * There are cases where ::rb_atomic_t is 32bit while ::VALUE is 64bit. This - * should be used for size related operations to support such platforms. - * - * @param var A variable of `void*`. - * @param oldval Expected value of `var` before the assignment. - * @param newval What you want to store at `var`. - * @retval oldval Successful assignment (`var` is now `newval`). - * @retval otherwise Something else is at `var`; not updated. - */ -#define RUBY_ATOMIC_VALUE_CAS(var, oldval, newval) \ - rbimpl_atomic_value_cas(&(var), (oldval), (newval)) - -/** @cond INTERNAL_MACRO */ -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline rb_atomic_t -rbimpl_atomic_fetch_add(volatile rb_atomic_t *ptr, rb_atomic_t val) -{ #if 0 +#elif defined HAVE_GCC_ATOMIC_BUILTINS +typedef unsigned int rb_atomic_t; +# define RUBY_ATOMIC_FETCH_ADD(var, val) __atomic_fetch_add(&(var), (val), __ATOMIC_SEQ_CST) +# define RUBY_ATOMIC_FETCH_SUB(var, val) __atomic_fetch_sub(&(var), (val), __ATOMIC_SEQ_CST) +# define RUBY_ATOMIC_OR(var, val) __atomic_fetch_or(&(var), (val), __ATOMIC_SEQ_CST) +# define RUBY_ATOMIC_EXCHANGE(var, val) __atomic_exchange_n(&(var), (val), __ATOMIC_SEQ_CST) +# define RUBY_ATOMIC_CAS(var, oldval, newval) RB_GNUC_EXTENSION_BLOCK( \ + __typeof__(var) oldvaldup = (oldval); /* oldval should not be modified */ \ + __atomic_compare_exchange_n(&(var), &oldvaldup, (newval), 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); \ + oldvaldup ) -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - return __atomic_fetch_add(ptr, val, __ATOMIC_SEQ_CST); +# define RUBY_ATOMIC_GENERIC_MACRO 1 -#elif defined(HAVE_GCC_SYNC_BUILTINS) - return __sync_fetch_and_add(ptr, val); +#elif defined HAVE_GCC_SYNC_BUILTINS +/* @shyouhei hack to support atomic operations in case of gcc. Gcc + * has its own pseudo-insns to support them. See info, or + * http://gcc.gnu.org/onlinedocs/gcc/Atomic-Builtins.html */ -#elif defined(_WIN32) - return InterlockedExchangeAdd(ptr, val); +typedef unsigned int rb_atomic_t; /* Anything OK */ +# define RUBY_ATOMIC_FETCH_ADD(var, val) __sync_fetch_and_add(&(var), (val)) +# define RUBY_ATOMIC_FETCH_SUB(var, val) __sync_fetch_and_sub(&(var), (val)) +# define RUBY_ATOMIC_OR(var, val) __sync_fetch_and_or(&(var), (val)) +# define RUBY_ATOMIC_EXCHANGE(var, val) __sync_lock_test_and_set(&(var), (val)) +# define RUBY_ATOMIC_CAS(var, oldval, newval) __sync_val_compare_and_swap(&(var), (oldval), (newval)) -#elif defined(__sun) && defined(HAVE_ATOMIC_H) - /* - * `atomic_add_int_nv` takes its second argument as `int`! Meanwhile our - * `rb_atomic_t` is unsigned. We cannot pass `val` as-is. We have to - * manually check integer overflow. - */ - RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX); - return atomic_add_int_nv(ptr, val) - val; +# define RUBY_ATOMIC_GENERIC_MACRO 1 -#else -# error Unsupported platform. +#elif defined _WIN32 +#if RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0) +#pragma intrinsic(_InterlockedOr) #endif -} +typedef LONG rb_atomic_t; -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) +# define RUBY_ATOMIC_SET(var, val) InterlockedExchange(&(var), (val)) +# define RUBY_ATOMIC_INC(var) InterlockedIncrement(&(var)) +# define RUBY_ATOMIC_DEC(var) InterlockedDecrement(&(var)) +# define RUBY_ATOMIC_FETCH_ADD(var, val) InterlockedExchangeAdd(&(var), (val)) +# define RUBY_ATOMIC_FETCH_SUB(var, val) InterlockedExchangeAdd(&(var), -(LONG)(val)) +#if defined __GNUC__ +# define RUBY_ATOMIC_OR(var, val) __asm__("lock\n\t" "orl\t%1, %0" : "=m"(var) : "Ir"(val)) +#elif RBIMPL_COMPILER_BEFORE(MSVC, 13, 0, 0) +# define RUBY_ATOMIC_OR(var, val) rb_w32_atomic_or(&(var), (val)) static inline void -rbimpl_atomic_add(volatile rb_atomic_t *ptr, rb_atomic_t val) +rb_w32_atomic_or(volatile rb_atomic_t *var, rb_atomic_t val) { -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - /* - * GCC on amd64 is smart enough to detect this `__atomic_add_fetch`'s - * return value is not used, then compiles it into single `LOCK ADD` - * instruction. - */ - __atomic_add_fetch(ptr, val, __ATOMIC_SEQ_CST); - -#elif defined(HAVE_GCC_SYNC_BUILTINS) - __sync_add_and_fetch(ptr, val); - -#elif defined(_WIN32) - /* - * `InterlockedExchangeAdd` is `LOCK XADD`. It seems there also is - * `_InterlockedAdd` intrinsic in ARM Windows but not for x86? Sticking to - * `InterlockedExchangeAdd` for better portability. - */ - InterlockedExchangeAdd(ptr, val); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) - /* Ditto for `atomic_add_int_nv`. */ - RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX); - atomic_add_int(ptr, val); - +#ifdef _M_IX86 + __asm mov eax, var; + __asm mov ecx, val; + __asm lock or [eax], ecx; #else -# error Unsupported platform. +#error unsupported architecture #endif } - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline void -rbimpl_atomic_size_add(volatile size_t *ptr, size_t val) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - __atomic_add_fetch(ptr, val, __ATOMIC_SEQ_CST); - -#elif defined(HAVE_GCC_SYNC_BUILTINS) - __sync_add_and_fetch(ptr, val); - -#elif defined(_WIN32) && defined(_M_AMD64) - /* Ditto for `InterlockeExchangedAdd`. */ - InterlockedExchangeAdd64(ptr, val); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx)) - /* Ditto for `atomic_add_int_nv`. */ - RBIMPL_ASSERT_OR_ASSUME(val <= LONG_MAX); - atomic_add_long(ptr, val); - #else - RBIMPL_STATIC_ASSERT(size_of_rb_atomic_t, sizeof *ptr == sizeof(rb_atomic_t)); - - volatile rb_atomic_t *const tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr); - rbimpl_atomic_add(tmp, val); - +# define RUBY_ATOMIC_OR(var, val) _InterlockedOr(&(var), (val)) #endif -} - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline void -rbimpl_atomic_inc(volatile rb_atomic_t *ptr) +# define RUBY_ATOMIC_EXCHANGE(var, val) InterlockedExchange(&(var), (val)) +# define RUBY_ATOMIC_CAS(var, oldval, newval) InterlockedCompareExchange(&(var), (newval), (oldval)) +# if RBIMPL_COMPILER_BEFORE(MSVC, 13, 0, 0) +static inline rb_atomic_t +rb_w32_atomic_cas(volatile rb_atomic_t *var, rb_atomic_t oldval, rb_atomic_t newval) { -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS) - rbimpl_atomic_add(ptr, 1); - -#elif defined(_WIN32) - InterlockedIncrement(ptr); + return (rb_atomic_t)InterlockedCompareExchange((PVOID *)var, (PVOID)newval, (PVOID)oldval); +} +# undef RUBY_ATOMIC_CAS +# define RUBY_ATOMIC_CAS(var, oldval, newval) rb_w32_atomic_cas(&(var), (oldval), (newval)) +# endif +# ifdef _M_AMD64 +# define RUBY_ATOMIC_SIZE_ADD(var, val) InterlockedExchangeAdd64((LONG_LONG *)&(var), (val)) +# define RUBY_ATOMIC_SIZE_SUB(var, val) InterlockedExchangeAdd64((LONG_LONG *)&(var), -(LONG)(val)) +# define RUBY_ATOMIC_SIZE_INC(var) InterlockedIncrement64(&(var)) +# define RUBY_ATOMIC_SIZE_DEC(var) InterlockedDecrement64(&(var)) +# define RUBY_ATOMIC_SIZE_EXCHANGE(var, val) InterlockedExchange64(&(var), (val)) +# define RUBY_ATOMIC_SIZE_CAS(var, oldval, newval) InterlockedCompareExchange64(&(var), (newval), (oldval)) +# else +# define RUBY_ATOMIC_SIZE_ADD(var, val) InterlockedExchangeAdd((LONG *)&(var), (val)) +# define RUBY_ATOMIC_SIZE_SUB(var, val) InterlockedExchangeAdd((LONG *)&(var), -(LONG)(val)) +# define RUBY_ATOMIC_SIZE_INC(var) InterlockedIncrement((LONG *)&(var)) +# define RUBY_ATOMIC_SIZE_DEC(var) InterlockedDecrement((LONG *)&(var)) +# define RUBY_ATOMIC_SIZE_EXCHANGE(var, val) InterlockedExchange((LONG *)&(var), (val)) +# endif + +# ifdef InterlockedExchangePointer +# define RUBY_ATOMIC_PTR_EXCHANGE(var, val) InterlockedExchangePointer((PVOID volatile *)&(var), (PVOID)(val)) +# endif /* See below for definitions of other situations */ #elif defined(__sun) && defined(HAVE_ATOMIC_H) - atomic_inc_uint(ptr); - -#else - rbimpl_atomic_add(ptr, 1); +#include <atomic.h> +typedef unsigned int rb_atomic_t; -#endif -} +# define RUBY_ATOMIC_INC(var) atomic_inc_uint(&(var)) +# define RUBY_ATOMIC_DEC(var) atomic_dec_uint(&(var)) +# define RUBY_ATOMIC_FETCH_ADD(var, val) rb_atomic_fetch_add(&(var), (val)) +# define RUBY_ATOMIC_FETCH_SUB(var, val) rb_atomic_fetch_sub(&(var), (val)) +# define RUBY_ATOMIC_ADD(var, val) atomic_add_uint(&(var), (val)) +# define RUBY_ATOMIC_SUB(var, val) atomic_sub_uint(&(var), (val)) +# define RUBY_ATOMIC_OR(var, val) atomic_or_uint(&(var), (val)) +# define RUBY_ATOMIC_EXCHANGE(var, val) atomic_swap_uint(&(var), (val)) +# define RUBY_ATOMIC_CAS(var, oldval, newval) atomic_cas_uint(&(var), (oldval), (newval)) -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline void -rbimpl_atomic_size_inc(volatile size_t *ptr) +static inline rb_atomic_t +rb_atomic_fetch_add(volatile rb_atomic_t *var, rb_atomic_t val) { -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS) - rbimpl_atomic_size_add(ptr, 1); - -#elif defined(_WIN32) && defined(_M_AMD64) - InterlockedIncrement64(ptr); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx)) - atomic_inc_ulong(ptr); - -#else - rbimpl_atomic_size_add(ptr, 1); - -#endif + return atomic_add_int_nv(var, val) - val; } -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) static inline rb_atomic_t -rbimpl_atomic_fetch_sub(volatile rb_atomic_t *ptr, rb_atomic_t val) +rb_atomic_fetch_sub(volatile rb_atomic_t *var, rb_atomic_t val) { -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - return __atomic_fetch_sub(ptr, val, __ATOMIC_SEQ_CST); - -#elif defined(HAVE_GCC_SYNC_BUILTINS) - return __sync_fetch_and_sub(ptr, val); - -#elif defined(_WIN32) - /* rb_atomic_t is signed here! Safe to do `-val`. */ - return InterlockedExchangeAdd(ptr, -val); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) - /* Ditto for `rbimpl_atomic_fetch_add`. */ - const signed neg = -1; - RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX); - return atomic_add_int_nv(ptr, neg * val) + val; - -#else -# error Unsupported platform. -#endif + return atomic_add_int_nv(var, (rb_atomic_t)(-(int)val)) + val; } -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline void -rbimpl_atomic_sub(volatile rb_atomic_t *ptr, rb_atomic_t val) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - __atomic_sub_fetch(ptr, val, __ATOMIC_SEQ_CST); - -#elif defined(HAVE_GCC_SYNC_BUILTINS) - __sync_sub_and_fetch(ptr, val); - -#elif defined(_WIN32) - InterlockedExchangeAdd(ptr, -val); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) - const signed neg = -1; - RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX); - atomic_add_int(ptr, neg * val); +# if defined(_LP64) || defined(_I32LPx) +# define RUBY_ATOMIC_SIZE_ADD(var, val) atomic_add_long(&(var), (val)) +# define RUBY_ATOMIC_SIZE_SUB(var, val) atomic_add_long(&(var), -(val)) +# define RUBY_ATOMIC_SIZE_INC(var) atomic_inc_ulong(&(var)) +# define RUBY_ATOMIC_SIZE_DEC(var) atomic_dec_ulong(&(var)) +# define RUBY_ATOMIC_SIZE_EXCHANGE(var, val) atomic_swap_ulong(&(var), (val)) +# define RUBY_ATOMIC_SIZE_CAS(var, oldval, val) atomic_cas_ulong(&(var), (oldval), (val)) +# else +# define RUBY_ATOMIC_SIZE_ADD(var, val) atomic_add_int(&(var), (val)) +# define RUBY_ATOMIC_SIZE_SUB(var, val) atomic_add_int(&(var), -(val)) +# define RUBY_ATOMIC_SIZE_INC(var) atomic_inc_uint(&(var)) +# define RUBY_ATOMIC_SIZE_DEC(var) atomic_dec_uint(&(var)) +# define RUBY_ATOMIC_SIZE_EXCHANGE(var, val) atomic_swap_uint(&(var), (val)) +# endif #else -# error Unsupported platform. +# error No atomic operation found #endif -} - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline void -rbimpl_atomic_size_sub(volatile size_t *ptr, size_t val) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - __atomic_sub_fetch(ptr, val, __ATOMIC_SEQ_CST); - -#elif defined(HAVE_GCC_SYNC_BUILTINS) - __sync_sub_and_fetch(ptr, val); - -#elif defined(_WIN32) && defined(_M_AMD64) - const ssize_t neg = -1; - InterlockedExchangeAdd64(ptr, neg * val); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx)) - const signed neg = -1; - RBIMPL_ASSERT_OR_ASSUME(val <= LONG_MAX); - atomic_add_long(ptr, neg * val); - -#else - RBIMPL_STATIC_ASSERT(size_of_rb_atomic_t, sizeof *ptr == sizeof(rb_atomic_t)); - - volatile rb_atomic_t *const tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr); - rbimpl_atomic_sub(tmp, val); +#ifndef RUBY_ATOMIC_SET +# define RUBY_ATOMIC_SET(var, val) (void)RUBY_ATOMIC_EXCHANGE(var, val) #endif -} - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline void -rbimpl_atomic_dec(volatile rb_atomic_t *ptr) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS) - rbimpl_atomic_sub(ptr, 1); - -#elif defined(_WIN32) - InterlockedDecrement(ptr); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) - atomic_dec_uint(ptr); - -#else - rbimpl_atomic_sub(ptr, 1); +#ifndef RUBY_ATOMIC_ADD +# define RUBY_ATOMIC_ADD(var, val) (void)RUBY_ATOMIC_FETCH_ADD(var, val) #endif -} - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline void -rbimpl_atomic_size_dec(volatile size_t *ptr) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS) - rbimpl_atomic_size_sub(ptr, 1); - -#elif defined(_WIN32) && defined(_M_AMD64) - InterlockedDecrement64(ptr); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx)) - atomic_dec_ulong(ptr); - -#else - rbimpl_atomic_size_sub(ptr, 1); +#ifndef RUBY_ATOMIC_SUB +# define RUBY_ATOMIC_SUB(var, val) (void)RUBY_ATOMIC_FETCH_SUB(var, val) #endif -} - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline void -rbimpl_atomic_or(volatile rb_atomic_t *ptr, rb_atomic_t val) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - __atomic_or_fetch(ptr, val, __ATOMIC_SEQ_CST); - -#elif defined(HAVE_GCC_SYNC_BUILTINS) - __sync_or_and_fetch(ptr, val); -#elif RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0) - _InterlockedOr(ptr, val); - -#elif defined(_WIN32) && defined(__GNUC__) - /* This was for old MinGW. Maybe not needed any longer? */ - __asm__( - "lock\n\t" - "orl\t%1, %0" - : "=m"(ptr) - : "Ir"(val)); - -#elif defined(_WIN32) && defined(_M_IX86) - __asm mov eax, ptr; - __asm mov ecx, val; - __asm lock or [eax], ecx; - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) - atomic_or_uint(ptr, val); - -#else -# error Unsupported platform. +#ifndef RUBY_ATOMIC_INC +# define RUBY_ATOMIC_INC(var) RUBY_ATOMIC_ADD(var, 1) #endif -} -/* Nobody uses this but for theoretical backwards compatibility... */ -#if RBIMPL_COMPILER_BEFORE(MSVC, 13, 0, 0) -static inline rb_atomic_t -rb_w32_atomic_or(volatile rb_atomic_t *var, rb_atomic_t val) -{ - return rbimpl_atomic_or(var, val); -} +#ifndef RUBY_ATOMIC_DEC +# define RUBY_ATOMIC_DEC(var) RUBY_ATOMIC_SUB(var, 1) #endif -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline rb_atomic_t -rbimpl_atomic_exchange(volatile rb_atomic_t *ptr, rb_atomic_t val) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - return __atomic_exchange_n(ptr, val, __ATOMIC_SEQ_CST); - -#elif defined(HAVE_GCC_SYNC_BUILTINS) - return __sync_lock_test_and_set(ptr, val); - -#elif defined(_WIN32) - return InterlockedExchange(ptr, val); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) - return atomic_swap_uint(ptr, val); - -#else -# error Unsupported platform. +#ifndef RUBY_ATOMIC_SIZE_INC +# define RUBY_ATOMIC_SIZE_INC(var) RUBY_ATOMIC_INC(var) #endif -} - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline size_t -rbimpl_atomic_size_exchange(volatile size_t *ptr, size_t val) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - return __atomic_exchange_n(ptr, val, __ATOMIC_SEQ_CST); - -#elif defined(HAVE_GCC_SYNC_BUILTINS) - return __sync_lock_test_and_set(ptr, val); - -#elif defined(_WIN32) && defined(_M_AMD64) - return InterlockedExchange64(ptr, val); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx)) - return atomic_swap_ulong(ptr, val); - -#else - RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof *ptr == sizeof(rb_atomic_t)); - - volatile rb_atomic_t *const tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr); - const rb_atomic_t ret = rbimpl_atomic_exchange(tmp, val); - return RBIMPL_CAST((size_t)ret); +#ifndef RUBY_ATOMIC_SIZE_DEC +# define RUBY_ATOMIC_SIZE_DEC(var) RUBY_ATOMIC_DEC(var) #endif -} - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline void * -rbimpl_atomic_ptr_exchange(void *volatile *ptr, const void *val) -{ -#if 0 - -#elif defined(InterlockedExchangePointer) - /* const_cast */ - PVOID *pptr = RBIMPL_CAST((PVOID *)ptr); - PVOID pval = RBIMPL_CAST((PVOID)val); - return InterlockedExchangePointer(pptr, pval); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) - return atomic_swap_ptr(ptr, RBIMPL_CAST((void *)val)); - -#else - RBIMPL_STATIC_ASSERT(sizeof_voidp, sizeof *ptr == sizeof(size_t)); - - const size_t sval = RBIMPL_CAST((size_t)val); - volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr); - const size_t sret = rbimpl_atomic_size_exchange(sptr, sval); - return RBIMPL_CAST((void *)sret); +#ifndef RUBY_ATOMIC_SIZE_EXCHANGE +# define RUBY_ATOMIC_SIZE_EXCHANGE(var, val) RUBY_ATOMIC_EXCHANGE(var, val) #endif -} -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline VALUE -rbimpl_atomic_value_exchange(volatile VALUE *ptr, VALUE val) -{ - RBIMPL_STATIC_ASSERT(sizeof_value, sizeof *ptr == sizeof(size_t)); - - const size_t sval = RBIMPL_CAST((size_t)val); - volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr); - const size_t sret = rbimpl_atomic_size_exchange(sptr, sval); - return RBIMPL_CAST((VALUE)sret); -} - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline rb_atomic_t -rbimpl_atomic_load(volatile rb_atomic_t *ptr) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); -#else - return rbimpl_atomic_fetch_add(ptr, 0); +#ifndef RUBY_ATOMIC_SIZE_CAS +# define RUBY_ATOMIC_SIZE_CAS(var, oldval, val) RUBY_ATOMIC_CAS(var, oldval, val) #endif -} - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline void -rbimpl_atomic_set(volatile rb_atomic_t *ptr, rb_atomic_t val) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - __atomic_store_n(ptr, val, __ATOMIC_SEQ_CST); - -#else - /* Maybe std::atomic<rb_atomic_t>::store can be faster? */ - rbimpl_atomic_exchange(ptr, val); +#ifndef RUBY_ATOMIC_SIZE_ADD +# define RUBY_ATOMIC_SIZE_ADD(var, val) RUBY_ATOMIC_ADD(var, val) #endif -} - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline rb_atomic_t -rbimpl_atomic_cas(volatile rb_atomic_t *ptr, rb_atomic_t oldval, rb_atomic_t newval) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - __atomic_compare_exchange_n( - ptr, &oldval, newval, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); - return oldval; -#elif defined(HAVE_GCC_SYNC_BUILTINS) - return __sync_val_compare_and_swap(ptr, oldval, newval); +#ifndef RUBY_ATOMIC_SIZE_SUB +# define RUBY_ATOMIC_SIZE_SUB(var, val) RUBY_ATOMIC_SUB(var, val) +#endif -#elif RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0) - return InterlockedCompareExchange(ptr, newval, oldval); +#if RUBY_ATOMIC_GENERIC_MACRO +# ifndef RUBY_ATOMIC_PTR_EXCHANGE +# define RUBY_ATOMIC_PTR_EXCHANGE(var, val) RUBY_ATOMIC_EXCHANGE(var, val) +# endif -#elif defined(_WIN32) - PVOID *pptr = RBIMPL_CAST((PVOID *)ptr); - PVOID pold = RBIMPL_CAST((PVOID)oldval); - PVOID pnew = RBIMPL_CAST((PVOID)newval); - PVOID pret = InterlockedCompareExchange(pptr, pnew, pold); - return RBIMPL_CAST((rb_atomic_t)pret); +# ifndef RUBY_ATOMIC_PTR_CAS +# define RUBY_ATOMIC_PTR_CAS(var, oldval, newval) RUBY_ATOMIC_CAS(var, oldval, newval) +# endif -#elif defined(__sun) && defined(HAVE_ATOMIC_H) - return atomic_cas_uint(ptr, oldval, newval); +# ifndef RUBY_ATOMIC_VALUE_EXCHANGE +# define RUBY_ATOMIC_VALUE_EXCHANGE(var, val) RUBY_ATOMIC_EXCHANGE(var, val) +# endif -#else -# error Unsupported platform. +# ifndef RUBY_ATOMIC_VALUE_CAS +# define RUBY_ATOMIC_VALUE_CAS(var, oldval, val) RUBY_ATOMIC_CAS(var, oldval, val) +# endif #endif -} -/* Nobody uses this but for theoretical backwards compatibility... */ -#if RBIMPL_COMPILER_BEFORE(MSVC, 13, 0, 0) -static inline rb_atomic_t -rb_w32_atomic_cas(volatile rb_atomic_t *var, rb_atomic_t oldval, rb_atomic_t newval) -{ - return rbimpl_atomic_cas(var, oldval, newval); -} +#ifndef RUBY_ATOMIC_PTR_EXCHANGE +# if SIZEOF_VOIDP == SIZEOF_SIZE_T +# define RUBY_ATOMIC_PTR_EXCHANGE(var, val) (void *)RUBY_ATOMIC_SIZE_EXCHANGE(*(size_t *)&(var), (size_t)(val)) +# else +# error No atomic exchange for void* +# endif #endif -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline size_t -rbimpl_atomic_size_cas(volatile size_t *ptr, size_t oldval, size_t newval) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - __atomic_compare_exchange_n( - ptr, &oldval, newval, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); - return oldval; - -#elif defined(HAVE_GCC_SYNC_BUILTINS) - return __sync_val_compare_and_swap(ptr, oldval, newval); - -#elif defined(_WIN32) && defined(_M_AMD64) - return InterlockedCompareExchange64(ptr, newval, oldval); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx)) - return atomic_cas_ulong(ptr, oldval, newval); - -#else - RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof *ptr == sizeof(rb_atomic_t)); - - volatile rb_atomic_t *tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr); - return rbimpl_atomic_cas(tmp, oldval, newval); - +#ifndef RUBY_ATOMIC_PTR_CAS +# if SIZEOF_VOIDP == SIZEOF_SIZE_T +# define RUBY_ATOMIC_PTR_CAS(var, oldval, val) (void *)RUBY_ATOMIC_SIZE_CAS(*(size_t *)&(var), (size_t)(oldval), (size_t)(val)) +# else +# error No atomic compare-and-set for void* +# endif #endif -} - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline void * -rbimpl_atomic_ptr_cas(void **ptr, const void *oldval, const void *newval) -{ -#if 0 - -#elif defined(InterlockedExchangePointer) - /* ... Can we say that InterlockedCompareExchangePtr surly exists when - * InterlockedExchangePointer is defined? Seems so but...?*/ - PVOID *pptr = RBIMPL_CAST((PVOID *)ptr); - PVOID pold = RBIMPL_CAST((PVOID)oldval); - PVOID pnew = RBIMPL_CAST((PVOID)newval); - return InterlockedCompareExchangePointer(pptr, pnew, pold); - -#elif defined(__sun) && defined(HAVE_ATOMIC_H) - void *pold = RBIMPL_CAST((void *)oldval); - void *pnew = RBIMPL_CAST((void *)newval); - return atomic_cas_ptr(ptr, pold, pnew); - - -#else - RBIMPL_STATIC_ASSERT(sizeof_voidp, sizeof *ptr == sizeof(size_t)); - - const size_t snew = RBIMPL_CAST((size_t)newval); - const size_t sold = RBIMPL_CAST((size_t)oldval); - volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr); - const size_t sret = rbimpl_atomic_size_cas(sptr, sold, snew); - return RBIMPL_CAST((void *)sret); +#ifndef RUBY_ATOMIC_VALUE_EXCHANGE +# if SIZEOF_VALUE == SIZEOF_SIZE_T +# define RUBY_ATOMIC_VALUE_EXCHANGE(var, val) RUBY_ATOMIC_SIZE_EXCHANGE(*(size_t *)&(var), (size_t)(val)) +# else +# error No atomic exchange for VALUE +# endif #endif -} -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline void * -rbimpl_atomic_ptr_load(void **ptr) -{ -#if 0 - -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) - return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); -#else - void *val = *ptr; - return rbimpl_atomic_ptr_cas(ptr, val, val); +#ifndef RUBY_ATOMIC_VALUE_CAS +# if SIZEOF_VALUE == SIZEOF_SIZE_T +# define RUBY_ATOMIC_VALUE_CAS(var, oldval, val) RUBY_ATOMIC_SIZE_CAS(*(size_t *)&(var), (size_t)(oldval), (size_t)(val)) +# else +# error No atomic compare-and-set for VALUE +# endif #endif -} - -RBIMPL_ATTR_ARTIFICIAL() -RBIMPL_ATTR_NOALIAS() -RBIMPL_ATTR_NONNULL((1)) -static inline VALUE -rbimpl_atomic_value_cas(volatile VALUE *ptr, VALUE oldval, VALUE newval) -{ - RBIMPL_STATIC_ASSERT(sizeof_value, sizeof *ptr == sizeof(size_t)); - const size_t snew = RBIMPL_CAST((size_t)newval); - const size_t sold = RBIMPL_CAST((size_t)oldval); - volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr); - const size_t sret = rbimpl_atomic_size_cas(sptr, sold, snew); - return RBIMPL_CAST((VALUE)sret); -} -/** @endcond */ #endif /* RUBY_ATOMIC_H */ |