mirror of
https://github.com/gcc-mirror/gcc.git
synced 2026-05-06 14:59:39 +02:00
5ced917508
- Some hardware has support for floating point atomic fetch_add (and similar). - There are existing compilers targetting this hardware that use libstdc++ -- e.g. NVC++. - Since the libstdc++ atomic<float>::fetch_add and similar is written directly as a CAS loop these compilers can not emit optimal code when seeing such constructs. - I hope to use __atomic_fetch_add builtins on floating point types directly in libstdc++ so these compilers can emit better code. - Clang already handles some floating point types in the __atomic_fetch_add family of builtins. - In order to only use this when available, I originally thought I could check against the resolved versions of the builtin in a manner something like `__has_builtin(__atomic_fetch_add_<fp-suffix>)`. I then realised that clang does not expose resolved versions of these atomic builtins to the user. From the clang discourse it was suggested we instead use SFINAE (which clang already supports). - I have recently pushed a patch for allowing the use of SFINAE on builtins: r15-6042-g9ed094a817ecaf Now that patch is committed, this patch does not change what happens for GCC, while it uses the builtin for codegen with clang. - I have previously sent a patchset upstream adding the ability to use __atomic_fetch_add and similar on floating point types. https://gcc.gnu.org/pipermail/gcc-patches/2024-November/668754.html Once that patchset is upstream (plus the automatic linking of libatomic as Joseph pointed out in the email below https://gcc.gnu.org/pipermail/gcc-patches/2024-October/665408.html ) then current GCC should start to use the builtin branch added in this patch. So *currently*, this patch allows external compilers (NVC++ in particular) to generate better code, and similarly lets clang understand the operation better since it maps to a known builtin. I hope that by GCC 16 this patch would also allow GCC to understand the operation better via mapping to a known builtin. libstdc++-v3/ChangeLog: * include/bits/atomic_base.h (__atomic_fetch_addable): Define new concept. (__atomic_impl::__fetch_add_flt): Use new concept to make use of __atomic_fetch_add when available. (__atomic_fetch_subtractable, __fetch_sub_flt): Likewise. (__atomic_add_fetchable, __add_fetch_flt): Likewise. (__atomic_sub_fetchable, __sub_fetch_flt): Likewise. Signed-off-by: Matthew Malcomson <mmalcomson@nvidia.com> Co-authored-by: Jonathan Wakely <jwakely@redhat.com>
2110 lines
62 KiB
C++
2110 lines
62 KiB
C++
// -*- C++ -*- header.
|
|
|
|
// Copyright (C) 2008-2025 Free Software Foundation, Inc.
|
|
//
|
|
// This file is part of the GNU ISO C++ Library. This library is free
|
|
// software; you can redistribute it and/or modify it under the
|
|
// terms of the GNU General Public License as published by the
|
|
// Free Software Foundation; either version 3, or (at your option)
|
|
// any later version.
|
|
|
|
// This library is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
|
|
// Under Section 7 of GPL version 3, you are granted additional
|
|
// permissions described in the GCC Runtime Library Exception, version
|
|
// 3.1, as published by the Free Software Foundation.
|
|
|
|
// You should have received a copy of the GNU General Public License and
|
|
// a copy of the GCC Runtime Library Exception along with this program;
|
|
// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
|
// <http://www.gnu.org/licenses/>.
|
|
|
|
/** @file bits/atomic_base.h
|
|
* This is an internal header file, included by other library headers.
|
|
* Do not attempt to use it directly. @headername{atomic}
|
|
*/
|
|
|
|
#ifndef _GLIBCXX_ATOMIC_BASE_H
|
|
#define _GLIBCXX_ATOMIC_BASE_H 1
|
|
|
|
#ifdef _GLIBCXX_SYSHDR
|
|
#pragma GCC system_header
|
|
#endif
|
|
|
|
#include <bits/c++config.h>
|
|
#include <new> // For placement new
|
|
#include <bits/atomic_lockfree_defines.h>
|
|
#include <bits/move.h>
|
|
|
|
#if __cplusplus > 201703L && _GLIBCXX_HOSTED
|
|
#include <bits/atomic_wait.h>
|
|
#endif
|
|
|
|
#ifndef _GLIBCXX_ALWAYS_INLINE
|
|
#define _GLIBCXX_ALWAYS_INLINE inline __attribute__((__always_inline__))
|
|
#endif
|
|
|
|
#include <bits/version.h>
|
|
|
|
namespace std _GLIBCXX_VISIBILITY(default)
|
|
{
|
|
_GLIBCXX_BEGIN_NAMESPACE_VERSION
|
|
|
|
/**
|
|
* @defgroup atomics Atomics
|
|
*
|
|
* Components for performing atomic operations.
|
|
* @{
|
|
*/
|
|
|
|
/// Enumeration for memory_order
|
|
#if __cplusplus > 201703L
|
|
enum class memory_order : int
|
|
{
|
|
relaxed,
|
|
consume,
|
|
acquire,
|
|
release,
|
|
acq_rel,
|
|
seq_cst
|
|
};
|
|
|
|
inline constexpr memory_order memory_order_relaxed = memory_order::relaxed;
|
|
inline constexpr memory_order memory_order_consume = memory_order::consume;
|
|
inline constexpr memory_order memory_order_acquire = memory_order::acquire;
|
|
inline constexpr memory_order memory_order_release = memory_order::release;
|
|
inline constexpr memory_order memory_order_acq_rel = memory_order::acq_rel;
|
|
inline constexpr memory_order memory_order_seq_cst = memory_order::seq_cst;
|
|
#else
|
|
enum memory_order : int
|
|
{
|
|
memory_order_relaxed,
|
|
memory_order_consume,
|
|
memory_order_acquire,
|
|
memory_order_release,
|
|
memory_order_acq_rel,
|
|
memory_order_seq_cst
|
|
};
|
|
#endif
|
|
|
|
/// @cond undocumented
|
|
enum __memory_order_modifier
|
|
{
|
|
__memory_order_mask = 0x0ffff,
|
|
__memory_order_modifier_mask = 0xffff0000,
|
|
__memory_order_hle_acquire = 0x10000,
|
|
__memory_order_hle_release = 0x20000
|
|
};
|
|
/// @endcond
|
|
|
|
constexpr memory_order
|
|
operator|(memory_order __m, __memory_order_modifier __mod) noexcept
|
|
{
|
|
return memory_order(int(__m) | int(__mod));
|
|
}
|
|
|
|
constexpr memory_order
|
|
operator&(memory_order __m, __memory_order_modifier __mod) noexcept
|
|
{
|
|
return memory_order(int(__m) & int(__mod));
|
|
}
|
|
|
|
/// @cond undocumented
|
|
|
|
// Drop release ordering as per [atomics.types.operations.req]/21
|
|
constexpr memory_order
|
|
__cmpexch_failure_order2(memory_order __m) noexcept
|
|
{
|
|
return __m == memory_order_acq_rel ? memory_order_acquire
|
|
: __m == memory_order_release ? memory_order_relaxed : __m;
|
|
}
|
|
|
|
constexpr memory_order
|
|
__cmpexch_failure_order(memory_order __m) noexcept
|
|
{
|
|
return memory_order(__cmpexch_failure_order2(__m & __memory_order_mask)
|
|
| __memory_order_modifier(__m & __memory_order_modifier_mask));
|
|
}
|
|
|
|
constexpr bool
|
|
__is_valid_cmpexch_failure_order(memory_order __m) noexcept
|
|
{
|
|
return (__m & __memory_order_mask) != memory_order_release
|
|
&& (__m & __memory_order_mask) != memory_order_acq_rel;
|
|
}
|
|
|
|
// Base types for atomics.
|
|
template<typename _IntTp>
|
|
struct __atomic_base;
|
|
|
|
/// @endcond
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
atomic_thread_fence(memory_order __m) noexcept
|
|
{ __atomic_thread_fence(int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
atomic_signal_fence(memory_order __m) noexcept
|
|
{ __atomic_signal_fence(int(__m)); }
|
|
|
|
/// kill_dependency
|
|
template<typename _Tp>
|
|
inline _Tp
|
|
kill_dependency(_Tp __y) noexcept
|
|
{
|
|
_Tp __ret(__y);
|
|
return __ret;
|
|
}
|
|
|
|
/// @cond undocumented
|
|
#if __glibcxx_atomic_value_initialization
|
|
# define _GLIBCXX20_INIT(I) = I
|
|
#else
|
|
# define _GLIBCXX20_INIT(I)
|
|
#endif
|
|
/// @endcond
|
|
|
|
#define ATOMIC_VAR_INIT(_VI) { _VI }
|
|
|
|
template<typename _Tp>
|
|
struct atomic;
|
|
|
|
template<typename _Tp>
|
|
struct atomic<_Tp*>;
|
|
|
|
/* The target's "set" value for test-and-set may not be exactly 1. */
|
|
#if __GCC_ATOMIC_TEST_AND_SET_TRUEVAL == 1
|
|
typedef bool __atomic_flag_data_type;
|
|
#else
|
|
typedef unsigned char __atomic_flag_data_type;
|
|
#endif
|
|
|
|
/// @cond undocumented
|
|
|
|
/*
|
|
* Base type for atomic_flag.
|
|
*
|
|
* Base type is POD with data, allowing atomic_flag to derive from
|
|
* it and meet the standard layout type requirement. In addition to
|
|
* compatibility with a C interface, this allows different
|
|
* implementations of atomic_flag to use the same atomic operation
|
|
* functions, via a standard conversion to the __atomic_flag_base
|
|
* argument.
|
|
*/
|
|
_GLIBCXX_BEGIN_EXTERN_C
|
|
|
|
struct __atomic_flag_base
|
|
{
|
|
__atomic_flag_data_type _M_i _GLIBCXX20_INIT({});
|
|
};
|
|
|
|
_GLIBCXX_END_EXTERN_C
|
|
|
|
/// @endcond
|
|
|
|
#define ATOMIC_FLAG_INIT { 0 }
|
|
|
|
/// atomic_flag
|
|
struct atomic_flag : public __atomic_flag_base
|
|
{
|
|
atomic_flag() noexcept = default;
|
|
~atomic_flag() noexcept = default;
|
|
atomic_flag(const atomic_flag&) = delete;
|
|
atomic_flag& operator=(const atomic_flag&) = delete;
|
|
atomic_flag& operator=(const atomic_flag&) volatile = delete;
|
|
|
|
// Conversion to ATOMIC_FLAG_INIT.
|
|
constexpr atomic_flag(bool __i) noexcept
|
|
: __atomic_flag_base{ _S_init(__i) }
|
|
{ }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
test_and_set(memory_order __m = memory_order_seq_cst) noexcept
|
|
{
|
|
return __atomic_test_and_set (&_M_i, int(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
test_and_set(memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{
|
|
return __atomic_test_and_set (&_M_i, int(__m));
|
|
}
|
|
|
|
#ifdef __glibcxx_atomic_flag_test // C++ >= 20
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
test(memory_order __m = memory_order_seq_cst) const noexcept
|
|
{
|
|
__atomic_flag_data_type __v;
|
|
__atomic_load(&_M_i, &__v, int(__m));
|
|
return __v == __GCC_ATOMIC_TEST_AND_SET_TRUEVAL;
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
test(memory_order __m = memory_order_seq_cst) const volatile noexcept
|
|
{
|
|
__atomic_flag_data_type __v;
|
|
__atomic_load(&_M_i, &__v, int(__m));
|
|
return __v == __GCC_ATOMIC_TEST_AND_SET_TRUEVAL;
|
|
}
|
|
#endif
|
|
|
|
#if __glibcxx_atomic_wait // C++ >= 20 && (linux_futex || gthread)
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
wait(bool __old,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{
|
|
const __atomic_flag_data_type __v
|
|
= __old ? __GCC_ATOMIC_TEST_AND_SET_TRUEVAL : 0;
|
|
|
|
std::__atomic_wait_address_v(&_M_i, __v,
|
|
[__m, this] { return __atomic_load_n(&_M_i, int(__m)); });
|
|
}
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_one() noexcept
|
|
{ std::__atomic_notify_address(&_M_i, false); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_all() noexcept
|
|
{ std::__atomic_notify_address(&_M_i, true); }
|
|
|
|
// TODO add const volatile overload
|
|
#endif // __glibcxx_atomic_wait
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
clear(memory_order __m = memory_order_seq_cst) noexcept
|
|
{
|
|
memory_order __b __attribute__ ((__unused__))
|
|
= __m & __memory_order_mask;
|
|
__glibcxx_assert(__b != memory_order_consume);
|
|
__glibcxx_assert(__b != memory_order_acquire);
|
|
__glibcxx_assert(__b != memory_order_acq_rel);
|
|
|
|
__atomic_clear (&_M_i, int(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
clear(memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{
|
|
memory_order __b __attribute__ ((__unused__))
|
|
= __m & __memory_order_mask;
|
|
__glibcxx_assert(__b != memory_order_consume);
|
|
__glibcxx_assert(__b != memory_order_acquire);
|
|
__glibcxx_assert(__b != memory_order_acq_rel);
|
|
|
|
__atomic_clear (&_M_i, int(__m));
|
|
}
|
|
|
|
private:
|
|
static constexpr __atomic_flag_data_type
|
|
_S_init(bool __i)
|
|
{ return __i ? __GCC_ATOMIC_TEST_AND_SET_TRUEVAL : 0; }
|
|
};
|
|
|
|
/// @cond undocumented
|
|
|
|
/// Base class for atomic integrals.
|
|
//
|
|
// For each of the integral types, define atomic_[integral type] struct
|
|
//
|
|
// atomic_bool bool
|
|
// atomic_char char
|
|
// atomic_schar signed char
|
|
// atomic_uchar unsigned char
|
|
// atomic_short short
|
|
// atomic_ushort unsigned short
|
|
// atomic_int int
|
|
// atomic_uint unsigned int
|
|
// atomic_long long
|
|
// atomic_ulong unsigned long
|
|
// atomic_llong long long
|
|
// atomic_ullong unsigned long long
|
|
// atomic_char8_t char8_t
|
|
// atomic_char16_t char16_t
|
|
// atomic_char32_t char32_t
|
|
// atomic_wchar_t wchar_t
|
|
//
|
|
// NB: Assuming _ITp is an integral scalar type that is 1, 2, 4, or
|
|
// 8 bytes, since that is what GCC built-in functions for atomic
|
|
// memory access expect.
|
|
template<typename _ITp>
|
|
struct __atomic_base
|
|
{
|
|
using value_type = _ITp;
|
|
using difference_type = value_type;
|
|
|
|
private:
|
|
typedef _ITp __int_type;
|
|
|
|
static constexpr int _S_alignment =
|
|
sizeof(_ITp) > alignof(_ITp) ? sizeof(_ITp) : alignof(_ITp);
|
|
|
|
alignas(_S_alignment) __int_type _M_i _GLIBCXX20_INIT(0);
|
|
|
|
public:
|
|
__atomic_base() noexcept = default;
|
|
~__atomic_base() noexcept = default;
|
|
__atomic_base(const __atomic_base&) = delete;
|
|
__atomic_base& operator=(const __atomic_base&) = delete;
|
|
__atomic_base& operator=(const __atomic_base&) volatile = delete;
|
|
|
|
// Requires __int_type convertible to _M_i.
|
|
constexpr __atomic_base(__int_type __i) noexcept : _M_i (__i) { }
|
|
|
|
operator __int_type() const noexcept
|
|
{ return load(); }
|
|
|
|
operator __int_type() const volatile noexcept
|
|
{ return load(); }
|
|
|
|
__int_type
|
|
operator=(__int_type __i) noexcept
|
|
{
|
|
store(__i);
|
|
return __i;
|
|
}
|
|
|
|
__int_type
|
|
operator=(__int_type __i) volatile noexcept
|
|
{
|
|
store(__i);
|
|
return __i;
|
|
}
|
|
|
|
__int_type
|
|
operator++(int) noexcept
|
|
{ return fetch_add(1); }
|
|
|
|
__int_type
|
|
operator++(int) volatile noexcept
|
|
{ return fetch_add(1); }
|
|
|
|
__int_type
|
|
operator--(int) noexcept
|
|
{ return fetch_sub(1); }
|
|
|
|
__int_type
|
|
operator--(int) volatile noexcept
|
|
{ return fetch_sub(1); }
|
|
|
|
__int_type
|
|
operator++() noexcept
|
|
{ return __atomic_add_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator++() volatile noexcept
|
|
{ return __atomic_add_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator--() noexcept
|
|
{ return __atomic_sub_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator--() volatile noexcept
|
|
{ return __atomic_sub_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator+=(__int_type __i) noexcept
|
|
{ return __atomic_add_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator+=(__int_type __i) volatile noexcept
|
|
{ return __atomic_add_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator-=(__int_type __i) noexcept
|
|
{ return __atomic_sub_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator-=(__int_type __i) volatile noexcept
|
|
{ return __atomic_sub_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator&=(__int_type __i) noexcept
|
|
{ return __atomic_and_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator&=(__int_type __i) volatile noexcept
|
|
{ return __atomic_and_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator|=(__int_type __i) noexcept
|
|
{ return __atomic_or_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator|=(__int_type __i) volatile noexcept
|
|
{ return __atomic_or_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator^=(__int_type __i) noexcept
|
|
{ return __atomic_xor_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
|
|
|
|
__int_type
|
|
operator^=(__int_type __i) volatile noexcept
|
|
{ return __atomic_xor_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
|
|
|
|
bool
|
|
is_lock_free() const noexcept
|
|
{
|
|
// Use a fake, minimally aligned pointer.
|
|
return __atomic_is_lock_free(sizeof(_M_i),
|
|
reinterpret_cast<void *>(-_S_alignment));
|
|
}
|
|
|
|
bool
|
|
is_lock_free() const volatile noexcept
|
|
{
|
|
// Use a fake, minimally aligned pointer.
|
|
return __atomic_is_lock_free(sizeof(_M_i),
|
|
reinterpret_cast<void *>(-_S_alignment));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
store(__int_type __i, memory_order __m = memory_order_seq_cst) noexcept
|
|
{
|
|
memory_order __b __attribute__ ((__unused__))
|
|
= __m & __memory_order_mask;
|
|
__glibcxx_assert(__b != memory_order_acquire);
|
|
__glibcxx_assert(__b != memory_order_acq_rel);
|
|
__glibcxx_assert(__b != memory_order_consume);
|
|
|
|
__atomic_store_n(&_M_i, __i, int(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
store(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{
|
|
memory_order __b __attribute__ ((__unused__))
|
|
= __m & __memory_order_mask;
|
|
__glibcxx_assert(__b != memory_order_acquire);
|
|
__glibcxx_assert(__b != memory_order_acq_rel);
|
|
__glibcxx_assert(__b != memory_order_consume);
|
|
|
|
__atomic_store_n(&_M_i, __i, int(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
load(memory_order __m = memory_order_seq_cst) const noexcept
|
|
{
|
|
memory_order __b __attribute__ ((__unused__))
|
|
= __m & __memory_order_mask;
|
|
__glibcxx_assert(__b != memory_order_release);
|
|
__glibcxx_assert(__b != memory_order_acq_rel);
|
|
|
|
return __atomic_load_n(&_M_i, int(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
load(memory_order __m = memory_order_seq_cst) const volatile noexcept
|
|
{
|
|
memory_order __b __attribute__ ((__unused__))
|
|
= __m & __memory_order_mask;
|
|
__glibcxx_assert(__b != memory_order_release);
|
|
__glibcxx_assert(__b != memory_order_acq_rel);
|
|
|
|
return __atomic_load_n(&_M_i, int(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
exchange(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{
|
|
return __atomic_exchange_n(&_M_i, __i, int(__m));
|
|
}
|
|
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
exchange(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{
|
|
return __atomic_exchange_n(&_M_i, __i, int(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_weak(__int_type& __i1, __int_type __i2,
|
|
memory_order __m1, memory_order __m2) noexcept
|
|
{
|
|
__glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
|
|
|
|
return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1,
|
|
int(__m1), int(__m2));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_weak(__int_type& __i1, __int_type __i2,
|
|
memory_order __m1,
|
|
memory_order __m2) volatile noexcept
|
|
{
|
|
__glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
|
|
|
|
return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1,
|
|
int(__m1), int(__m2));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_weak(__int_type& __i1, __int_type __i2,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{
|
|
return compare_exchange_weak(__i1, __i2, __m,
|
|
__cmpexch_failure_order(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_weak(__int_type& __i1, __int_type __i2,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{
|
|
return compare_exchange_weak(__i1, __i2, __m,
|
|
__cmpexch_failure_order(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_strong(__int_type& __i1, __int_type __i2,
|
|
memory_order __m1, memory_order __m2) noexcept
|
|
{
|
|
__glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
|
|
|
|
return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0,
|
|
int(__m1), int(__m2));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_strong(__int_type& __i1, __int_type __i2,
|
|
memory_order __m1,
|
|
memory_order __m2) volatile noexcept
|
|
{
|
|
__glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
|
|
|
|
return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0,
|
|
int(__m1), int(__m2));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_strong(__int_type& __i1, __int_type __i2,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{
|
|
return compare_exchange_strong(__i1, __i2, __m,
|
|
__cmpexch_failure_order(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_strong(__int_type& __i1, __int_type __i2,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{
|
|
return compare_exchange_strong(__i1, __i2, __m,
|
|
__cmpexch_failure_order(__m));
|
|
}
|
|
|
|
#if __glibcxx_atomic_wait
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
wait(__int_type __old,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{
|
|
std::__atomic_wait_address_v(&_M_i, __old,
|
|
[__m, this] { return this->load(__m); });
|
|
}
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_one() noexcept
|
|
{ std::__atomic_notify_address(&_M_i, false); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_all() noexcept
|
|
{ std::__atomic_notify_address(&_M_i, true); }
|
|
|
|
// TODO add const volatile overload
|
|
#endif // __glibcxx_atomic_wait
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
fetch_add(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{ return __atomic_fetch_add(&_M_i, __i, int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
fetch_add(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{ return __atomic_fetch_add(&_M_i, __i, int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
fetch_sub(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{ return __atomic_fetch_sub(&_M_i, __i, int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
fetch_sub(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{ return __atomic_fetch_sub(&_M_i, __i, int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
fetch_and(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{ return __atomic_fetch_and(&_M_i, __i, int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
fetch_and(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{ return __atomic_fetch_and(&_M_i, __i, int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
fetch_or(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{ return __atomic_fetch_or(&_M_i, __i, int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
fetch_or(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{ return __atomic_fetch_or(&_M_i, __i, int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
fetch_xor(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{ return __atomic_fetch_xor(&_M_i, __i, int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __int_type
|
|
fetch_xor(__int_type __i,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{ return __atomic_fetch_xor(&_M_i, __i, int(__m)); }
|
|
};
|
|
|
|
|
|
/// Partial specialization for pointer types.
|
|
template<typename _PTp>
|
|
struct __atomic_base<_PTp*>
|
|
{
|
|
private:
|
|
typedef _PTp* __pointer_type;
|
|
|
|
__pointer_type _M_p _GLIBCXX20_INIT(nullptr);
|
|
|
|
static constexpr ptrdiff_t
|
|
_S_type_size(ptrdiff_t __d)
|
|
{ return __d * sizeof(_PTp); }
|
|
|
|
public:
|
|
__atomic_base() noexcept = default;
|
|
~__atomic_base() noexcept = default;
|
|
__atomic_base(const __atomic_base&) = delete;
|
|
__atomic_base& operator=(const __atomic_base&) = delete;
|
|
__atomic_base& operator=(const __atomic_base&) volatile = delete;
|
|
|
|
// Requires __pointer_type convertible to _M_p.
|
|
constexpr __atomic_base(__pointer_type __p) noexcept : _M_p (__p) { }
|
|
|
|
operator __pointer_type() const noexcept
|
|
{ return load(); }
|
|
|
|
operator __pointer_type() const volatile noexcept
|
|
{ return load(); }
|
|
|
|
__pointer_type
|
|
operator=(__pointer_type __p) noexcept
|
|
{
|
|
store(__p);
|
|
return __p;
|
|
}
|
|
|
|
__pointer_type
|
|
operator=(__pointer_type __p) volatile noexcept
|
|
{
|
|
store(__p);
|
|
return __p;
|
|
}
|
|
|
|
__pointer_type
|
|
operator++(int) noexcept
|
|
{ return fetch_add(1); }
|
|
|
|
__pointer_type
|
|
operator++(int) volatile noexcept
|
|
{ return fetch_add(1); }
|
|
|
|
__pointer_type
|
|
operator--(int) noexcept
|
|
{ return fetch_sub(1); }
|
|
|
|
__pointer_type
|
|
operator--(int) volatile noexcept
|
|
{ return fetch_sub(1); }
|
|
|
|
__pointer_type
|
|
operator++() noexcept
|
|
{ return __atomic_add_fetch(&_M_p, _S_type_size(1),
|
|
int(memory_order_seq_cst)); }
|
|
|
|
__pointer_type
|
|
operator++() volatile noexcept
|
|
{ return __atomic_add_fetch(&_M_p, _S_type_size(1),
|
|
int(memory_order_seq_cst)); }
|
|
|
|
__pointer_type
|
|
operator--() noexcept
|
|
{ return __atomic_sub_fetch(&_M_p, _S_type_size(1),
|
|
int(memory_order_seq_cst)); }
|
|
|
|
__pointer_type
|
|
operator--() volatile noexcept
|
|
{ return __atomic_sub_fetch(&_M_p, _S_type_size(1),
|
|
int(memory_order_seq_cst)); }
|
|
|
|
__pointer_type
|
|
operator+=(ptrdiff_t __d) noexcept
|
|
{ return __atomic_add_fetch(&_M_p, _S_type_size(__d),
|
|
int(memory_order_seq_cst)); }
|
|
|
|
__pointer_type
|
|
operator+=(ptrdiff_t __d) volatile noexcept
|
|
{ return __atomic_add_fetch(&_M_p, _S_type_size(__d),
|
|
int(memory_order_seq_cst)); }
|
|
|
|
__pointer_type
|
|
operator-=(ptrdiff_t __d) noexcept
|
|
{ return __atomic_sub_fetch(&_M_p, _S_type_size(__d),
|
|
int(memory_order_seq_cst)); }
|
|
|
|
__pointer_type
|
|
operator-=(ptrdiff_t __d) volatile noexcept
|
|
{ return __atomic_sub_fetch(&_M_p, _S_type_size(__d),
|
|
int(memory_order_seq_cst)); }
|
|
|
|
bool
|
|
is_lock_free() const noexcept
|
|
{
|
|
// Produce a fake, minimally aligned pointer.
|
|
return __atomic_is_lock_free(sizeof(_M_p),
|
|
reinterpret_cast<void *>(-__alignof(_M_p)));
|
|
}
|
|
|
|
bool
|
|
is_lock_free() const volatile noexcept
|
|
{
|
|
// Produce a fake, minimally aligned pointer.
|
|
return __atomic_is_lock_free(sizeof(_M_p),
|
|
reinterpret_cast<void *>(-__alignof(_M_p)));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
store(__pointer_type __p,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{
|
|
memory_order __b __attribute__ ((__unused__))
|
|
= __m & __memory_order_mask;
|
|
|
|
__glibcxx_assert(__b != memory_order_acquire);
|
|
__glibcxx_assert(__b != memory_order_acq_rel);
|
|
__glibcxx_assert(__b != memory_order_consume);
|
|
|
|
__atomic_store_n(&_M_p, __p, int(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
store(__pointer_type __p,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{
|
|
memory_order __b __attribute__ ((__unused__))
|
|
= __m & __memory_order_mask;
|
|
__glibcxx_assert(__b != memory_order_acquire);
|
|
__glibcxx_assert(__b != memory_order_acq_rel);
|
|
__glibcxx_assert(__b != memory_order_consume);
|
|
|
|
__atomic_store_n(&_M_p, __p, int(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __pointer_type
|
|
load(memory_order __m = memory_order_seq_cst) const noexcept
|
|
{
|
|
memory_order __b __attribute__ ((__unused__))
|
|
= __m & __memory_order_mask;
|
|
__glibcxx_assert(__b != memory_order_release);
|
|
__glibcxx_assert(__b != memory_order_acq_rel);
|
|
|
|
return __atomic_load_n(&_M_p, int(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __pointer_type
|
|
load(memory_order __m = memory_order_seq_cst) const volatile noexcept
|
|
{
|
|
memory_order __b __attribute__ ((__unused__))
|
|
= __m & __memory_order_mask;
|
|
__glibcxx_assert(__b != memory_order_release);
|
|
__glibcxx_assert(__b != memory_order_acq_rel);
|
|
|
|
return __atomic_load_n(&_M_p, int(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __pointer_type
|
|
exchange(__pointer_type __p,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{
|
|
return __atomic_exchange_n(&_M_p, __p, int(__m));
|
|
}
|
|
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __pointer_type
|
|
exchange(__pointer_type __p,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{
|
|
return __atomic_exchange_n(&_M_p, __p, int(__m));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
|
|
memory_order __m1,
|
|
memory_order __m2) noexcept
|
|
{
|
|
__glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
|
|
|
|
return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 1,
|
|
int(__m1), int(__m2));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
|
|
memory_order __m1,
|
|
memory_order __m2) volatile noexcept
|
|
{
|
|
__glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
|
|
|
|
return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 1,
|
|
int(__m1), int(__m2));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
|
|
memory_order __m1,
|
|
memory_order __m2) noexcept
|
|
{
|
|
__glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
|
|
|
|
return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0,
|
|
int(__m1), int(__m2));
|
|
}
|
|
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
|
|
memory_order __m1,
|
|
memory_order __m2) volatile noexcept
|
|
{
|
|
__glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
|
|
|
|
return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0,
|
|
int(__m1), int(__m2));
|
|
}
|
|
|
|
#if __glibcxx_atomic_wait
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
wait(__pointer_type __old,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{
|
|
std::__atomic_wait_address_v(&_M_p, __old,
|
|
[__m, this]
|
|
{ return this->load(__m); });
|
|
}
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_one() const noexcept
|
|
{ std::__atomic_notify_address(&_M_p, false); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_all() const noexcept
|
|
{ std::__atomic_notify_address(&_M_p, true); }
|
|
|
|
// TODO add const volatile overload
|
|
#endif // __glibcxx_atomic_wait
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __pointer_type
|
|
fetch_add(ptrdiff_t __d,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{ return __atomic_fetch_add(&_M_p, _S_type_size(__d), int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __pointer_type
|
|
fetch_add(ptrdiff_t __d,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{ return __atomic_fetch_add(&_M_p, _S_type_size(__d), int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __pointer_type
|
|
fetch_sub(ptrdiff_t __d,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{ return __atomic_fetch_sub(&_M_p, _S_type_size(__d), int(__m)); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE __pointer_type
|
|
fetch_sub(ptrdiff_t __d,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{ return __atomic_fetch_sub(&_M_p, _S_type_size(__d), int(__m)); }
|
|
};
|
|
|
|
namespace __atomic_impl
|
|
{
|
|
// Implementation details of atomic padding handling
|
|
|
|
template<typename _Tp>
|
|
constexpr bool
|
|
__maybe_has_padding()
|
|
{
|
|
#if ! __has_builtin(__builtin_clear_padding)
|
|
return false;
|
|
#elif __has_builtin(__has_unique_object_representations)
|
|
return !__has_unique_object_representations(_Tp)
|
|
&& !is_same<_Tp, float>::value && !is_same<_Tp, double>::value;
|
|
#else
|
|
return true;
|
|
#endif
|
|
}
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _GLIBCXX14_CONSTEXPR _Tp*
|
|
__clear_padding(_Tp& __val) noexcept
|
|
{
|
|
auto* __ptr = std::__addressof(__val);
|
|
#if __has_builtin(__builtin_clear_padding)
|
|
if _GLIBCXX17_CONSTEXPR (__atomic_impl::__maybe_has_padding<_Tp>())
|
|
__builtin_clear_padding(__ptr);
|
|
#endif
|
|
return __ptr;
|
|
}
|
|
|
|
// Remove volatile and create a non-deduced context for value arguments.
|
|
template<typename _Tp>
|
|
using _Val = typename remove_volatile<_Tp>::type;
|
|
|
|
#pragma GCC diagnostic push
|
|
#pragma GCC diagnostic ignored "-Wc++17-extensions"
|
|
|
|
template<bool _AtomicRef = false, typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
__compare_exchange(_Tp& __val, _Val<_Tp>& __e, _Val<_Tp>& __i,
|
|
bool __is_weak,
|
|
memory_order __s, memory_order __f) noexcept
|
|
{
|
|
__glibcxx_assert(__is_valid_cmpexch_failure_order(__f));
|
|
|
|
using _Vp = _Val<_Tp>;
|
|
_Tp* const __pval = std::__addressof(__val);
|
|
|
|
if constexpr (!__atomic_impl::__maybe_has_padding<_Vp>())
|
|
{
|
|
return __atomic_compare_exchange(__pval, std::__addressof(__e),
|
|
std::__addressof(__i), __is_weak,
|
|
int(__s), int(__f));
|
|
}
|
|
else if constexpr (!_AtomicRef) // std::atomic<T>
|
|
{
|
|
// Clear padding of the value we want to set:
|
|
_Vp* const __pi = __atomic_impl::__clear_padding(__i);
|
|
// Only allowed to modify __e on failure, so make a copy:
|
|
_Vp __exp = __e;
|
|
// Clear padding of the expected value:
|
|
_Vp* const __pexp = __atomic_impl::__clear_padding(__exp);
|
|
|
|
// For std::atomic<T> we know that the contained value will already
|
|
// have zeroed padding, so trivial memcmp semantics are OK.
|
|
if (__atomic_compare_exchange(__pval, __pexp, __pi,
|
|
__is_weak, int(__s), int(__f)))
|
|
return true;
|
|
// Value bits must be different, copy from __exp back to __e:
|
|
__builtin_memcpy(std::__addressof(__e), __pexp, sizeof(_Vp));
|
|
return false;
|
|
}
|
|
else // std::atomic_ref<T> where T has padding bits.
|
|
{
|
|
// Clear padding of the value we want to set:
|
|
_Vp* const __pi = __atomic_impl::__clear_padding(__i);
|
|
|
|
// Only allowed to modify __e on failure, so make a copy:
|
|
_Vp __exp = __e;
|
|
// Optimistically assume that a previous store had zeroed padding
|
|
// so that zeroing it in the expected value will match first time.
|
|
_Vp* const __pexp = __atomic_impl::__clear_padding(__exp);
|
|
|
|
// compare_exchange is specified to compare value representations.
|
|
// Need to check whether a failure is 'real' or just due to
|
|
// differences in padding bits. This loop should run no more than
|
|
// three times, because the worst case scenario is:
|
|
// First CAS fails because the actual value has non-zero padding.
|
|
// Second CAS fails because another thread stored the same value,
|
|
// but now with padding cleared. Third CAS succeeds.
|
|
// We will never need to loop a fourth time, because any value
|
|
// written by another thread (whether via store, exchange or
|
|
// compare_exchange) will have had its padding cleared.
|
|
while (true)
|
|
{
|
|
// Copy of the expected value so we can clear its padding.
|
|
_Vp __orig = __exp;
|
|
|
|
if (__atomic_compare_exchange(__pval, __pexp, __pi,
|
|
__is_weak, int(__s), int(__f)))
|
|
return true;
|
|
|
|
// Copy of the actual value so we can clear its padding.
|
|
_Vp __curr = __exp;
|
|
|
|
// Compare value representations (i.e. ignoring padding).
|
|
if (__builtin_memcmp(__atomic_impl::__clear_padding(__orig),
|
|
__atomic_impl::__clear_padding(__curr),
|
|
sizeof(_Vp)))
|
|
{
|
|
// Value representations compare unequal, real failure.
|
|
__builtin_memcpy(std::__addressof(__e), __pexp,
|
|
sizeof(_Vp));
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
#pragma GCC diagnostic pop
|
|
} // namespace __atomic_impl
|
|
|
|
#if __cplusplus > 201703L
|
|
// Implementation details of atomic_ref and atomic<floating-point>.
|
|
namespace __atomic_impl
|
|
{
|
|
// Like _Val<T> above, but for difference_type arguments.
|
|
template<typename _Tp>
|
|
using _Diff = __conditional_t<is_pointer_v<_Tp>, ptrdiff_t, _Val<_Tp>>;
|
|
|
|
template<size_t _Size, size_t _Align>
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
is_lock_free() noexcept
|
|
{
|
|
// Produce a fake, minimally aligned pointer.
|
|
return __atomic_is_lock_free(_Size, reinterpret_cast<void *>(-_Align));
|
|
}
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
store(_Tp* __ptr, _Val<_Tp> __t, memory_order __m) noexcept
|
|
{
|
|
__atomic_store(__ptr, __atomic_impl::__clear_padding(__t), int(__m));
|
|
}
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _Val<_Tp>
|
|
load(const _Tp* __ptr, memory_order __m) noexcept
|
|
{
|
|
alignas(_Tp) unsigned char __buf[sizeof(_Tp)];
|
|
auto* __dest = reinterpret_cast<_Val<_Tp>*>(__buf);
|
|
__atomic_load(__ptr, __dest, int(__m));
|
|
return *__dest;
|
|
}
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _Val<_Tp>
|
|
exchange(_Tp* __ptr, _Val<_Tp> __desired, memory_order __m) noexcept
|
|
{
|
|
alignas(_Tp) unsigned char __buf[sizeof(_Tp)];
|
|
auto* __dest = reinterpret_cast<_Val<_Tp>*>(__buf);
|
|
__atomic_exchange(__ptr, __atomic_impl::__clear_padding(__desired),
|
|
__dest, int(__m));
|
|
return *__dest;
|
|
}
|
|
|
|
template<bool _AtomicRef = false, typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_weak(_Tp* __ptr, _Val<_Tp>& __expected,
|
|
_Val<_Tp> __desired, memory_order __success,
|
|
memory_order __failure) noexcept
|
|
{
|
|
return __atomic_impl::__compare_exchange<_AtomicRef>(
|
|
*__ptr, __expected, __desired, true, __success, __failure);
|
|
}
|
|
|
|
template<bool _AtomicRef = false, typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE bool
|
|
compare_exchange_strong(_Tp* __ptr, _Val<_Tp>& __expected,
|
|
_Val<_Tp> __desired, memory_order __success,
|
|
memory_order __failure) noexcept
|
|
{
|
|
return __atomic_impl::__compare_exchange<_AtomicRef>(
|
|
*__ptr, __expected, __desired, false, __success, __failure);
|
|
}
|
|
|
|
#if __glibcxx_atomic_wait
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
wait(const _Tp* __ptr, _Val<_Tp> __old,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{
|
|
std::__atomic_wait_address_v(__ptr, __old,
|
|
[__ptr, __m]() { return __atomic_impl::load(__ptr, __m); });
|
|
}
|
|
|
|
// TODO add const volatile overload
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_one(const _Tp* __ptr) noexcept
|
|
{ std::__atomic_notify_address(__ptr, false); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_all(const _Tp* __ptr) noexcept
|
|
{ std::__atomic_notify_address(__ptr, true); }
|
|
|
|
// TODO add const volatile overload
|
|
#endif // __glibcxx_atomic_wait
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _Tp
|
|
fetch_add(_Tp* __ptr, _Diff<_Tp> __i, memory_order __m) noexcept
|
|
{ return __atomic_fetch_add(__ptr, __i, int(__m)); }
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _Tp
|
|
fetch_sub(_Tp* __ptr, _Diff<_Tp> __i, memory_order __m) noexcept
|
|
{ return __atomic_fetch_sub(__ptr, __i, int(__m)); }
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _Tp
|
|
fetch_and(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
|
|
{ return __atomic_fetch_and(__ptr, __i, int(__m)); }
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _Tp
|
|
fetch_or(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
|
|
{ return __atomic_fetch_or(__ptr, __i, int(__m)); }
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _Tp
|
|
fetch_xor(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
|
|
{ return __atomic_fetch_xor(__ptr, __i, int(__m)); }
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _Tp
|
|
__add_fetch(_Tp* __ptr, _Diff<_Tp> __i) noexcept
|
|
{ return __atomic_add_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _Tp
|
|
__sub_fetch(_Tp* __ptr, _Diff<_Tp> __i) noexcept
|
|
{ return __atomic_sub_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _Tp
|
|
__and_fetch(_Tp* __ptr, _Val<_Tp> __i) noexcept
|
|
{ return __atomic_and_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _Tp
|
|
__or_fetch(_Tp* __ptr, _Val<_Tp> __i) noexcept
|
|
{ return __atomic_or_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
|
|
|
|
template<typename _Tp>
|
|
_GLIBCXX_ALWAYS_INLINE _Tp
|
|
__xor_fetch(_Tp* __ptr, _Val<_Tp> __i) noexcept
|
|
{ return __atomic_xor_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
|
|
|
|
template<typename _Tp>
|
|
concept __atomic_fetch_addable
|
|
= requires (_Tp __t) { __atomic_fetch_add(&__t, __t, 0); };
|
|
|
|
template<typename _Tp>
|
|
_Tp
|
|
__fetch_add_flt(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
|
|
{
|
|
if constexpr (__atomic_fetch_addable<_Tp>)
|
|
return __atomic_fetch_add(__ptr, __i, int(__m));
|
|
else
|
|
{
|
|
_Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
|
|
_Val<_Tp> __newval = __oldval + __i;
|
|
while (!compare_exchange_weak (__ptr, __oldval, __newval, __m,
|
|
memory_order_relaxed))
|
|
__newval = __oldval + __i;
|
|
return __oldval;
|
|
}
|
|
}
|
|
|
|
template<typename _Tp>
|
|
concept __atomic_fetch_subtractable
|
|
= requires (_Tp __t) { __atomic_fetch_sub(&__t, __t, 0); };
|
|
|
|
template<typename _Tp>
|
|
_Tp
|
|
__fetch_sub_flt(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
|
|
{
|
|
if constexpr (__atomic_fetch_subtractable<_Tp>)
|
|
return __atomic_fetch_sub(__ptr, __i, int(__m));
|
|
else
|
|
{
|
|
_Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
|
|
_Val<_Tp> __newval = __oldval - __i;
|
|
while (!compare_exchange_weak (__ptr, __oldval, __newval, __m,
|
|
memory_order_relaxed))
|
|
__newval = __oldval - __i;
|
|
return __oldval;
|
|
}
|
|
}
|
|
|
|
template<typename _Tp>
|
|
concept __atomic_add_fetchable
|
|
= requires (_Tp __t) { __atomic_add_fetch(&__t, __t, 0); };
|
|
|
|
template<typename _Tp>
|
|
_Tp
|
|
__add_fetch_flt(_Tp* __ptr, _Val<_Tp> __i) noexcept
|
|
{
|
|
if constexpr (__atomic_add_fetchable<_Tp>)
|
|
return __atomic_add_fetch(__ptr, __i, __ATOMIC_SEQ_CST);
|
|
else
|
|
{
|
|
_Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
|
|
_Val<_Tp> __newval = __oldval + __i;
|
|
while (!compare_exchange_weak (__ptr, __oldval, __newval,
|
|
memory_order_seq_cst,
|
|
memory_order_relaxed))
|
|
__newval = __oldval + __i;
|
|
return __newval;
|
|
}
|
|
}
|
|
|
|
template<typename _Tp>
|
|
concept __atomic_sub_fetchable
|
|
= requires (_Tp __t) { __atomic_sub_fetch(&__t, __t, 0); };
|
|
|
|
template<typename _Tp>
|
|
_Tp
|
|
__sub_fetch_flt(_Tp* __ptr, _Val<_Tp> __i) noexcept
|
|
{
|
|
if constexpr (__atomic_sub_fetchable<_Tp>)
|
|
return __atomic_sub_fetch(__ptr, __i, __ATOMIC_SEQ_CST);
|
|
else
|
|
{
|
|
_Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
|
|
_Val<_Tp> __newval = __oldval - __i;
|
|
while (!compare_exchange_weak (__ptr, __oldval, __newval,
|
|
memory_order_seq_cst,
|
|
memory_order_relaxed))
|
|
__newval = __oldval - __i;
|
|
return __newval;
|
|
}
|
|
}
|
|
} // namespace __atomic_impl
|
|
|
|
// base class for atomic<floating-point-type>
|
|
template<typename _Fp>
|
|
struct __atomic_float
|
|
{
|
|
static_assert(is_floating_point_v<_Fp>);
|
|
|
|
static constexpr size_t _S_alignment = __alignof__(_Fp);
|
|
|
|
public:
|
|
using value_type = _Fp;
|
|
using difference_type = value_type;
|
|
|
|
static constexpr bool is_always_lock_free
|
|
= __atomic_always_lock_free(sizeof(_Fp), 0);
|
|
|
|
__atomic_float() = default;
|
|
|
|
constexpr
|
|
__atomic_float(_Fp __t) : _M_fp(__t)
|
|
{ __atomic_impl::__clear_padding(_M_fp); }
|
|
|
|
__atomic_float(const __atomic_float&) = delete;
|
|
__atomic_float& operator=(const __atomic_float&) = delete;
|
|
__atomic_float& operator=(const __atomic_float&) volatile = delete;
|
|
|
|
_Fp
|
|
operator=(_Fp __t) volatile noexcept
|
|
{
|
|
this->store(__t);
|
|
return __t;
|
|
}
|
|
|
|
_Fp
|
|
operator=(_Fp __t) noexcept
|
|
{
|
|
this->store(__t);
|
|
return __t;
|
|
}
|
|
|
|
bool
|
|
is_lock_free() const volatile noexcept
|
|
{ return __atomic_impl::is_lock_free<sizeof(_Fp), _S_alignment>(); }
|
|
|
|
bool
|
|
is_lock_free() const noexcept
|
|
{ return __atomic_impl::is_lock_free<sizeof(_Fp), _S_alignment>(); }
|
|
|
|
void
|
|
store(_Fp __t, memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{ __atomic_impl::store(&_M_fp, __t, __m); }
|
|
|
|
void
|
|
store(_Fp __t, memory_order __m = memory_order_seq_cst) noexcept
|
|
{ __atomic_impl::store(&_M_fp, __t, __m); }
|
|
|
|
_Fp
|
|
load(memory_order __m = memory_order_seq_cst) const volatile noexcept
|
|
{ return __atomic_impl::load(&_M_fp, __m); }
|
|
|
|
_Fp
|
|
load(memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::load(&_M_fp, __m); }
|
|
|
|
operator _Fp() const volatile noexcept { return this->load(); }
|
|
operator _Fp() const noexcept { return this->load(); }
|
|
|
|
_Fp
|
|
exchange(_Fp __desired,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{ return __atomic_impl::exchange(&_M_fp, __desired, __m); }
|
|
|
|
_Fp
|
|
exchange(_Fp __desired,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{ return __atomic_impl::exchange(&_M_fp, __desired, __m); }
|
|
|
|
bool
|
|
compare_exchange_weak(_Fp& __expected, _Fp __desired,
|
|
memory_order __success,
|
|
memory_order __failure) noexcept
|
|
{
|
|
return __atomic_impl::compare_exchange_weak(&_M_fp,
|
|
__expected, __desired,
|
|
__success, __failure);
|
|
}
|
|
|
|
bool
|
|
compare_exchange_weak(_Fp& __expected, _Fp __desired,
|
|
memory_order __success,
|
|
memory_order __failure) volatile noexcept
|
|
{
|
|
return __atomic_impl::compare_exchange_weak(&_M_fp,
|
|
__expected, __desired,
|
|
__success, __failure);
|
|
}
|
|
|
|
bool
|
|
compare_exchange_strong(_Fp& __expected, _Fp __desired,
|
|
memory_order __success,
|
|
memory_order __failure) noexcept
|
|
{
|
|
return __atomic_impl::compare_exchange_strong(&_M_fp,
|
|
__expected, __desired,
|
|
__success, __failure);
|
|
}
|
|
|
|
bool
|
|
compare_exchange_strong(_Fp& __expected, _Fp __desired,
|
|
memory_order __success,
|
|
memory_order __failure) volatile noexcept
|
|
{
|
|
return __atomic_impl::compare_exchange_strong(&_M_fp,
|
|
__expected, __desired,
|
|
__success, __failure);
|
|
}
|
|
|
|
bool
|
|
compare_exchange_weak(_Fp& __expected, _Fp __desired,
|
|
memory_order __order = memory_order_seq_cst)
|
|
noexcept
|
|
{
|
|
return compare_exchange_weak(__expected, __desired, __order,
|
|
__cmpexch_failure_order(__order));
|
|
}
|
|
|
|
bool
|
|
compare_exchange_weak(_Fp& __expected, _Fp __desired,
|
|
memory_order __order = memory_order_seq_cst)
|
|
volatile noexcept
|
|
{
|
|
return compare_exchange_weak(__expected, __desired, __order,
|
|
__cmpexch_failure_order(__order));
|
|
}
|
|
|
|
bool
|
|
compare_exchange_strong(_Fp& __expected, _Fp __desired,
|
|
memory_order __order = memory_order_seq_cst)
|
|
noexcept
|
|
{
|
|
return compare_exchange_strong(__expected, __desired, __order,
|
|
__cmpexch_failure_order(__order));
|
|
}
|
|
|
|
bool
|
|
compare_exchange_strong(_Fp& __expected, _Fp __desired,
|
|
memory_order __order = memory_order_seq_cst)
|
|
volatile noexcept
|
|
{
|
|
return compare_exchange_strong(__expected, __desired, __order,
|
|
__cmpexch_failure_order(__order));
|
|
}
|
|
|
|
#if __glibcxx_atomic_wait
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ __atomic_impl::wait(&_M_fp, __old, __m); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_one() const noexcept
|
|
{ __atomic_impl::notify_one(&_M_fp); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_all() const noexcept
|
|
{ __atomic_impl::notify_all(&_M_fp); }
|
|
|
|
// TODO add const volatile overload
|
|
#endif // __glibcxx_atomic_wait
|
|
|
|
value_type
|
|
fetch_add(value_type __i,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{ return __atomic_impl::__fetch_add_flt(&_M_fp, __i, __m); }
|
|
|
|
value_type
|
|
fetch_add(value_type __i,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{ return __atomic_impl::__fetch_add_flt(&_M_fp, __i, __m); }
|
|
|
|
value_type
|
|
fetch_sub(value_type __i,
|
|
memory_order __m = memory_order_seq_cst) noexcept
|
|
{ return __atomic_impl::__fetch_sub_flt(&_M_fp, __i, __m); }
|
|
|
|
value_type
|
|
fetch_sub(value_type __i,
|
|
memory_order __m = memory_order_seq_cst) volatile noexcept
|
|
{ return __atomic_impl::__fetch_sub_flt(&_M_fp, __i, __m); }
|
|
|
|
value_type
|
|
operator+=(value_type __i) noexcept
|
|
{ return __atomic_impl::__add_fetch_flt(&_M_fp, __i); }
|
|
|
|
value_type
|
|
operator+=(value_type __i) volatile noexcept
|
|
{ return __atomic_impl::__add_fetch_flt(&_M_fp, __i); }
|
|
|
|
value_type
|
|
operator-=(value_type __i) noexcept
|
|
{ return __atomic_impl::__sub_fetch_flt(&_M_fp, __i); }
|
|
|
|
value_type
|
|
operator-=(value_type __i) volatile noexcept
|
|
{ return __atomic_impl::__sub_fetch_flt(&_M_fp, __i); }
|
|
|
|
private:
|
|
alignas(_S_alignment) _Fp _M_fp _GLIBCXX20_INIT(0);
|
|
};
|
|
#undef _GLIBCXX20_INIT
|
|
|
|
template<typename _Tp,
|
|
bool = is_integral_v<_Tp> && !is_same_v<_Tp, bool>,
|
|
bool = is_floating_point_v<_Tp>>
|
|
struct __atomic_ref;
|
|
|
|
// base class for non-integral, non-floating-point, non-pointer types
|
|
template<typename _Tp>
|
|
struct __atomic_ref<_Tp, false, false>
|
|
{
|
|
static_assert(is_trivially_copyable_v<_Tp>);
|
|
|
|
// 1/2/4/8/16-byte types must be aligned to at least their size.
|
|
static constexpr int _S_min_alignment
|
|
= (sizeof(_Tp) & (sizeof(_Tp) - 1)) || sizeof(_Tp) > 16
|
|
? 0 : sizeof(_Tp);
|
|
|
|
public:
|
|
using value_type = _Tp;
|
|
|
|
static constexpr bool is_always_lock_free
|
|
= __atomic_always_lock_free(sizeof(_Tp), 0);
|
|
|
|
static constexpr size_t required_alignment
|
|
= _S_min_alignment > alignof(_Tp) ? _S_min_alignment : alignof(_Tp);
|
|
|
|
__atomic_ref& operator=(const __atomic_ref&) = delete;
|
|
|
|
explicit
|
|
__atomic_ref(_Tp& __t) : _M_ptr(std::__addressof(__t))
|
|
{
|
|
__glibcxx_assert(((__UINTPTR_TYPE__)_M_ptr % required_alignment) == 0);
|
|
}
|
|
|
|
__atomic_ref(const __atomic_ref&) noexcept = default;
|
|
|
|
_Tp
|
|
operator=(_Tp __t) const noexcept
|
|
{
|
|
this->store(__t);
|
|
return __t;
|
|
}
|
|
|
|
operator _Tp() const noexcept { return this->load(); }
|
|
|
|
bool
|
|
is_lock_free() const noexcept
|
|
{ return __atomic_impl::is_lock_free<sizeof(_Tp), required_alignment>(); }
|
|
|
|
void
|
|
store(_Tp __t, memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ __atomic_impl::store(_M_ptr, __t, __m); }
|
|
|
|
_Tp
|
|
load(memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::load(_M_ptr, __m); }
|
|
|
|
_Tp
|
|
exchange(_Tp __desired, memory_order __m = memory_order_seq_cst)
|
|
const noexcept
|
|
{ return __atomic_impl::exchange(_M_ptr, __desired, __m); }
|
|
|
|
bool
|
|
compare_exchange_weak(_Tp& __expected, _Tp __desired,
|
|
memory_order __success,
|
|
memory_order __failure) const noexcept
|
|
{
|
|
return __atomic_impl::compare_exchange_weak<true>(
|
|
_M_ptr, __expected, __desired, __success, __failure);
|
|
}
|
|
|
|
bool
|
|
compare_exchange_strong(_Tp& __expected, _Tp __desired,
|
|
memory_order __success,
|
|
memory_order __failure) const noexcept
|
|
{
|
|
return __atomic_impl::compare_exchange_strong<true>(
|
|
_M_ptr, __expected, __desired, __success, __failure);
|
|
}
|
|
|
|
bool
|
|
compare_exchange_weak(_Tp& __expected, _Tp __desired,
|
|
memory_order __order = memory_order_seq_cst)
|
|
const noexcept
|
|
{
|
|
return compare_exchange_weak(__expected, __desired, __order,
|
|
__cmpexch_failure_order(__order));
|
|
}
|
|
|
|
bool
|
|
compare_exchange_strong(_Tp& __expected, _Tp __desired,
|
|
memory_order __order = memory_order_seq_cst)
|
|
const noexcept
|
|
{
|
|
return compare_exchange_strong(__expected, __desired, __order,
|
|
__cmpexch_failure_order(__order));
|
|
}
|
|
|
|
#if __glibcxx_atomic_wait
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ __atomic_impl::wait(_M_ptr, __old, __m); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_one() const noexcept
|
|
{ __atomic_impl::notify_one(_M_ptr); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_all() const noexcept
|
|
{ __atomic_impl::notify_all(_M_ptr); }
|
|
|
|
// TODO add const volatile overload
|
|
#endif // __glibcxx_atomic_wait
|
|
|
|
private:
|
|
_Tp* _M_ptr;
|
|
};
|
|
|
|
// base class for atomic_ref<integral-type>
|
|
template<typename _Tp>
|
|
struct __atomic_ref<_Tp, true, false>
|
|
{
|
|
static_assert(is_integral_v<_Tp>);
|
|
|
|
public:
|
|
using value_type = _Tp;
|
|
using difference_type = value_type;
|
|
|
|
static constexpr bool is_always_lock_free
|
|
= __atomic_always_lock_free(sizeof(_Tp), 0);
|
|
|
|
static constexpr size_t required_alignment
|
|
= sizeof(_Tp) > alignof(_Tp) ? sizeof(_Tp) : alignof(_Tp);
|
|
|
|
__atomic_ref() = delete;
|
|
__atomic_ref& operator=(const __atomic_ref&) = delete;
|
|
|
|
explicit
|
|
__atomic_ref(_Tp& __t) : _M_ptr(&__t)
|
|
{
|
|
__glibcxx_assert(((__UINTPTR_TYPE__)_M_ptr % required_alignment) == 0);
|
|
}
|
|
|
|
__atomic_ref(const __atomic_ref&) noexcept = default;
|
|
|
|
_Tp
|
|
operator=(_Tp __t) const noexcept
|
|
{
|
|
this->store(__t);
|
|
return __t;
|
|
}
|
|
|
|
operator _Tp() const noexcept { return this->load(); }
|
|
|
|
bool
|
|
is_lock_free() const noexcept
|
|
{
|
|
return __atomic_impl::is_lock_free<sizeof(_Tp), required_alignment>();
|
|
}
|
|
|
|
void
|
|
store(_Tp __t, memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ __atomic_impl::store(_M_ptr, __t, __m); }
|
|
|
|
_Tp
|
|
load(memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::load(_M_ptr, __m); }
|
|
|
|
_Tp
|
|
exchange(_Tp __desired,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::exchange(_M_ptr, __desired, __m); }
|
|
|
|
bool
|
|
compare_exchange_weak(_Tp& __expected, _Tp __desired,
|
|
memory_order __success,
|
|
memory_order __failure) const noexcept
|
|
{
|
|
return __atomic_impl::compare_exchange_weak<true>(
|
|
_M_ptr, __expected, __desired, __success, __failure);
|
|
}
|
|
|
|
bool
|
|
compare_exchange_strong(_Tp& __expected, _Tp __desired,
|
|
memory_order __success,
|
|
memory_order __failure) const noexcept
|
|
{
|
|
return __atomic_impl::compare_exchange_strong<true>(
|
|
_M_ptr, __expected, __desired, __success, __failure);
|
|
}
|
|
|
|
bool
|
|
compare_exchange_weak(_Tp& __expected, _Tp __desired,
|
|
memory_order __order = memory_order_seq_cst)
|
|
const noexcept
|
|
{
|
|
return compare_exchange_weak(__expected, __desired, __order,
|
|
__cmpexch_failure_order(__order));
|
|
}
|
|
|
|
bool
|
|
compare_exchange_strong(_Tp& __expected, _Tp __desired,
|
|
memory_order __order = memory_order_seq_cst)
|
|
const noexcept
|
|
{
|
|
return compare_exchange_strong(__expected, __desired, __order,
|
|
__cmpexch_failure_order(__order));
|
|
}
|
|
|
|
#if __glibcxx_atomic_wait
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ __atomic_impl::wait(_M_ptr, __old, __m); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_one() const noexcept
|
|
{ __atomic_impl::notify_one(_M_ptr); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_all() const noexcept
|
|
{ __atomic_impl::notify_all(_M_ptr); }
|
|
|
|
// TODO add const volatile overload
|
|
#endif // __glibcxx_atomic_wait
|
|
|
|
value_type
|
|
fetch_add(value_type __i,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::fetch_add(_M_ptr, __i, __m); }
|
|
|
|
value_type
|
|
fetch_sub(value_type __i,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::fetch_sub(_M_ptr, __i, __m); }
|
|
|
|
value_type
|
|
fetch_and(value_type __i,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::fetch_and(_M_ptr, __i, __m); }
|
|
|
|
value_type
|
|
fetch_or(value_type __i,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::fetch_or(_M_ptr, __i, __m); }
|
|
|
|
value_type
|
|
fetch_xor(value_type __i,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::fetch_xor(_M_ptr, __i, __m); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE value_type
|
|
operator++(int) const noexcept
|
|
{ return fetch_add(1); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE value_type
|
|
operator--(int) const noexcept
|
|
{ return fetch_sub(1); }
|
|
|
|
value_type
|
|
operator++() const noexcept
|
|
{ return __atomic_impl::__add_fetch(_M_ptr, value_type(1)); }
|
|
|
|
value_type
|
|
operator--() const noexcept
|
|
{ return __atomic_impl::__sub_fetch(_M_ptr, value_type(1)); }
|
|
|
|
value_type
|
|
operator+=(value_type __i) const noexcept
|
|
{ return __atomic_impl::__add_fetch(_M_ptr, __i); }
|
|
|
|
value_type
|
|
operator-=(value_type __i) const noexcept
|
|
{ return __atomic_impl::__sub_fetch(_M_ptr, __i); }
|
|
|
|
value_type
|
|
operator&=(value_type __i) const noexcept
|
|
{ return __atomic_impl::__and_fetch(_M_ptr, __i); }
|
|
|
|
value_type
|
|
operator|=(value_type __i) const noexcept
|
|
{ return __atomic_impl::__or_fetch(_M_ptr, __i); }
|
|
|
|
value_type
|
|
operator^=(value_type __i) const noexcept
|
|
{ return __atomic_impl::__xor_fetch(_M_ptr, __i); }
|
|
|
|
private:
|
|
_Tp* _M_ptr;
|
|
};
|
|
|
|
// base class for atomic_ref<floating-point-type>
|
|
template<typename _Fp>
|
|
struct __atomic_ref<_Fp, false, true>
|
|
{
|
|
static_assert(is_floating_point_v<_Fp>);
|
|
|
|
public:
|
|
using value_type = _Fp;
|
|
using difference_type = value_type;
|
|
|
|
static constexpr bool is_always_lock_free
|
|
= __atomic_always_lock_free(sizeof(_Fp), 0);
|
|
|
|
static constexpr size_t required_alignment = __alignof__(_Fp);
|
|
|
|
__atomic_ref() = delete;
|
|
__atomic_ref& operator=(const __atomic_ref&) = delete;
|
|
|
|
explicit
|
|
__atomic_ref(_Fp& __t) : _M_ptr(&__t)
|
|
{
|
|
__glibcxx_assert(((__UINTPTR_TYPE__)_M_ptr % required_alignment) == 0);
|
|
}
|
|
|
|
__atomic_ref(const __atomic_ref&) noexcept = default;
|
|
|
|
_Fp
|
|
operator=(_Fp __t) const noexcept
|
|
{
|
|
this->store(__t);
|
|
return __t;
|
|
}
|
|
|
|
operator _Fp() const noexcept { return this->load(); }
|
|
|
|
bool
|
|
is_lock_free() const noexcept
|
|
{
|
|
return __atomic_impl::is_lock_free<sizeof(_Fp), required_alignment>();
|
|
}
|
|
|
|
void
|
|
store(_Fp __t, memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ __atomic_impl::store(_M_ptr, __t, __m); }
|
|
|
|
_Fp
|
|
load(memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::load(_M_ptr, __m); }
|
|
|
|
_Fp
|
|
exchange(_Fp __desired,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::exchange(_M_ptr, __desired, __m); }
|
|
|
|
bool
|
|
compare_exchange_weak(_Fp& __expected, _Fp __desired,
|
|
memory_order __success,
|
|
memory_order __failure) const noexcept
|
|
{
|
|
return __atomic_impl::compare_exchange_weak<true>(
|
|
_M_ptr, __expected, __desired, __success, __failure);
|
|
}
|
|
|
|
bool
|
|
compare_exchange_strong(_Fp& __expected, _Fp __desired,
|
|
memory_order __success,
|
|
memory_order __failure) const noexcept
|
|
{
|
|
return __atomic_impl::compare_exchange_strong<true>(
|
|
_M_ptr, __expected, __desired, __success, __failure);
|
|
}
|
|
|
|
bool
|
|
compare_exchange_weak(_Fp& __expected, _Fp __desired,
|
|
memory_order __order = memory_order_seq_cst)
|
|
const noexcept
|
|
{
|
|
return compare_exchange_weak(__expected, __desired, __order,
|
|
__cmpexch_failure_order(__order));
|
|
}
|
|
|
|
bool
|
|
compare_exchange_strong(_Fp& __expected, _Fp __desired,
|
|
memory_order __order = memory_order_seq_cst)
|
|
const noexcept
|
|
{
|
|
return compare_exchange_strong(__expected, __desired, __order,
|
|
__cmpexch_failure_order(__order));
|
|
}
|
|
|
|
#if __glibcxx_atomic_wait
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ __atomic_impl::wait(_M_ptr, __old, __m); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_one() const noexcept
|
|
{ __atomic_impl::notify_one(_M_ptr); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_all() const noexcept
|
|
{ __atomic_impl::notify_all(_M_ptr); }
|
|
|
|
// TODO add const volatile overload
|
|
#endif // __glibcxx_atomic_wait
|
|
|
|
value_type
|
|
fetch_add(value_type __i,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::__fetch_add_flt(_M_ptr, __i, __m); }
|
|
|
|
value_type
|
|
fetch_sub(value_type __i,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::__fetch_sub_flt(_M_ptr, __i, __m); }
|
|
|
|
value_type
|
|
operator+=(value_type __i) const noexcept
|
|
{ return __atomic_impl::__add_fetch_flt(_M_ptr, __i); }
|
|
|
|
value_type
|
|
operator-=(value_type __i) const noexcept
|
|
{ return __atomic_impl::__sub_fetch_flt(_M_ptr, __i); }
|
|
|
|
private:
|
|
_Fp* _M_ptr;
|
|
};
|
|
|
|
// base class for atomic_ref<pointer-type>
|
|
template<typename _Tp>
|
|
struct __atomic_ref<_Tp*, false, false>
|
|
{
|
|
public:
|
|
using value_type = _Tp*;
|
|
using difference_type = ptrdiff_t;
|
|
|
|
static constexpr bool is_always_lock_free = ATOMIC_POINTER_LOCK_FREE == 2;
|
|
|
|
static constexpr size_t required_alignment = __alignof__(_Tp*);
|
|
|
|
__atomic_ref() = delete;
|
|
__atomic_ref& operator=(const __atomic_ref&) = delete;
|
|
|
|
explicit
|
|
__atomic_ref(_Tp*& __t) : _M_ptr(std::__addressof(__t))
|
|
{
|
|
__glibcxx_assert(((__UINTPTR_TYPE__)_M_ptr % required_alignment) == 0);
|
|
}
|
|
|
|
__atomic_ref(const __atomic_ref&) noexcept = default;
|
|
|
|
_Tp*
|
|
operator=(_Tp* __t) const noexcept
|
|
{
|
|
this->store(__t);
|
|
return __t;
|
|
}
|
|
|
|
operator _Tp*() const noexcept { return this->load(); }
|
|
|
|
bool
|
|
is_lock_free() const noexcept
|
|
{
|
|
return __atomic_impl::is_lock_free<sizeof(_Tp*), required_alignment>();
|
|
}
|
|
|
|
void
|
|
store(_Tp* __t, memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ __atomic_impl::store(_M_ptr, __t, __m); }
|
|
|
|
_Tp*
|
|
load(memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::load(_M_ptr, __m); }
|
|
|
|
_Tp*
|
|
exchange(_Tp* __desired,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::exchange(_M_ptr, __desired, __m); }
|
|
|
|
bool
|
|
compare_exchange_weak(_Tp*& __expected, _Tp* __desired,
|
|
memory_order __success,
|
|
memory_order __failure) const noexcept
|
|
{
|
|
return __atomic_impl::compare_exchange_weak<true>(
|
|
_M_ptr, __expected, __desired, __success, __failure);
|
|
}
|
|
|
|
bool
|
|
compare_exchange_strong(_Tp*& __expected, _Tp* __desired,
|
|
memory_order __success,
|
|
memory_order __failure) const noexcept
|
|
{
|
|
return __atomic_impl::compare_exchange_strong<true>(
|
|
_M_ptr, __expected, __desired, __success, __failure);
|
|
}
|
|
|
|
bool
|
|
compare_exchange_weak(_Tp*& __expected, _Tp* __desired,
|
|
memory_order __order = memory_order_seq_cst)
|
|
const noexcept
|
|
{
|
|
return compare_exchange_weak(__expected, __desired, __order,
|
|
__cmpexch_failure_order(__order));
|
|
}
|
|
|
|
bool
|
|
compare_exchange_strong(_Tp*& __expected, _Tp* __desired,
|
|
memory_order __order = memory_order_seq_cst)
|
|
const noexcept
|
|
{
|
|
return compare_exchange_strong(__expected, __desired, __order,
|
|
__cmpexch_failure_order(__order));
|
|
}
|
|
|
|
#if __glibcxx_atomic_wait
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
wait(_Tp* __old, memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ __atomic_impl::wait(_M_ptr, __old, __m); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_one() const noexcept
|
|
{ __atomic_impl::notify_one(_M_ptr); }
|
|
|
|
// TODO add const volatile overload
|
|
|
|
_GLIBCXX_ALWAYS_INLINE void
|
|
notify_all() const noexcept
|
|
{ __atomic_impl::notify_all(_M_ptr); }
|
|
|
|
// TODO add const volatile overload
|
|
#endif // __glibcxx_atomic_wait
|
|
|
|
_GLIBCXX_ALWAYS_INLINE value_type
|
|
fetch_add(difference_type __d,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::fetch_add(_M_ptr, _S_type_size(__d), __m); }
|
|
|
|
_GLIBCXX_ALWAYS_INLINE value_type
|
|
fetch_sub(difference_type __d,
|
|
memory_order __m = memory_order_seq_cst) const noexcept
|
|
{ return __atomic_impl::fetch_sub(_M_ptr, _S_type_size(__d), __m); }
|
|
|
|
value_type
|
|
operator++(int) const noexcept
|
|
{ return fetch_add(1); }
|
|
|
|
value_type
|
|
operator--(int) const noexcept
|
|
{ return fetch_sub(1); }
|
|
|
|
value_type
|
|
operator++() const noexcept
|
|
{
|
|
return __atomic_impl::__add_fetch(_M_ptr, _S_type_size(1));
|
|
}
|
|
|
|
value_type
|
|
operator--() const noexcept
|
|
{
|
|
return __atomic_impl::__sub_fetch(_M_ptr, _S_type_size(1));
|
|
}
|
|
|
|
value_type
|
|
operator+=(difference_type __d) const noexcept
|
|
{
|
|
return __atomic_impl::__add_fetch(_M_ptr, _S_type_size(__d));
|
|
}
|
|
|
|
value_type
|
|
operator-=(difference_type __d) const noexcept
|
|
{
|
|
return __atomic_impl::__sub_fetch(_M_ptr, _S_type_size(__d));
|
|
}
|
|
|
|
private:
|
|
static constexpr ptrdiff_t
|
|
_S_type_size(ptrdiff_t __d) noexcept
|
|
{
|
|
static_assert(is_object_v<_Tp>);
|
|
return __d * sizeof(_Tp);
|
|
}
|
|
|
|
_Tp** _M_ptr;
|
|
};
|
|
#endif // C++2a
|
|
|
|
/// @endcond
|
|
|
|
/// @} group atomics
|
|
|
|
_GLIBCXX_END_NAMESPACE_VERSION
|
|
} // namespace std
|
|
|
|
#endif
|