libstdc++: Add platform wait functions for FreeBSD [PR120527]

This defines __platform_wait, __platform_notify, and
__platform_wait_until for FreeBSD, making use of the _umtx_op syscall.

The Linux versions of those functions only support 32-bit integers, but
the FreeBSD versions use the syscall for both 32-bit and 64-bit types,
as the _umtx_op supports both.

We also need to change __spin_impl because it currently assumes the
waitable at args._M_obj is always a __platform_wait_t. Because FreeBSD
supports waiting on both 32-bit and 64-bit integers, we need a
platform-specific function for loading a value from _M_obj.  This adds a
new __platform_load function, which does an atomic load of the right
size. The Linux definition just loads an int, but for FreeBSD it depends
on _M_obj_size. We also need a generic version of the function for
platforms without __platform_wait, because __spin_impl is always used,
even when the __waitable_state contains a condition_variable.

libstdc++-v3/ChangeLog:

	PR libstdc++/120527
	* include/bits/atomic_wait.h [__FreeBSD__] (__platform_wait_t):
	Define typedef.
	[__FreeBSD__] (__platform_wait_uses_type): Define variable
	template.
	* src/c++20/atomic.cc (__platform_load): New function.
	[__FreeBSD__] (_GLIBCXX_HAVE_PLATFORM_WAIT, __platform_wait)
	(__platform_notify, __platform_wait_until): Define.
	(__detail::__spin_impl): Use __platform_load.

Reviewed-by: Tomasz Kamiński <tkaminsk@redhat.com>

libstdc++-v3/include/bits/atomic_wait.h

@@ -69,6 +69,17 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     inline constexpr bool __platform_wait_uses_type
       = __detail::__waitable<_Tp>
 	  && sizeof(_Tp) == sizeof(int) && alignof(_Tp) >= 4;
+#elif defined __FreeBSD__ && __SIZEOF_LONG__ == 8
+  namespace __detail
+  {
+    using __platform_wait_t = __UINT64_TYPE__;
+    inline constexpr size_t __platform_wait_alignment = 8;
+  }
+  template<typename _Tp>
+    inline constexpr bool __platform_wait_uses_type
+      = __detail::__waitable<_Tp>
+	  && ((sizeof(_Tp) == 4 && alignof(_Tp) >= 4)
+		|| (sizeof(_Tp) == 8 && alignof(_Tp) >= 8));
 #else
 // define _GLIBCX_HAVE_PLATFORM_WAIT and implement __platform_wait()
 // and __platform_notify() if there is a more efficient primitive supported

libstdc++-v3/src/c++20/atomic.cc

@@ -27,7 +27,7 @@
 #if __glibcxx_atomic_wait
 #include <atomic>
 #include <bits/atomic_timed_wait.h>
-#include <cstdint> // uint32_t, uint64_t
+#include <cstdint> // uint32_t, uint64_t, uintptr_t
 #include <climits> // INT_MAX
 #include <cerrno>  // errno, ETIMEDOUT, etc.
 #include <bits/std_mutex.h>  // std::mutex, std::__condvar
@@ -39,6 +39,11 @@
 # include <unistd.h>
 # include <sys/time.h> // timespec
 # define _GLIBCXX_HAVE_PLATFORM_WAIT 1
+#elif defined __FreeBSD__ && __FreeBSD__ >= 11 && __SIZEOF_LONG__ == 8
+# include <sys/types.h>
+# include <sys/umtx.h>
+# include <sys/time.h>
+# define _GLIBCXX_HAVE_PLATFORM_WAIT 1
 #endif
 
 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
@@ -87,6 +92,13 @@ namespace
 			__wait_clock_t::time_point timeout,
 			int obj_size) = delete;
 
+  // This is needed even when we don't have __platform_wait
+  [[gnu::always_inline]]
+  inline __wait_value_type
+  __platform_load(const __platform_wait_t* addr, int memory_order,
+		  int /* obj_sz */) noexcept
+  { return __atomic_load_n(addr, memory_order); }
+
 #elif defined _GLIBCXX_HAVE_LINUX_FUTEX
 
   const int futex_private_flag = 128;
@@ -136,6 +148,68 @@ namespace
       }
     return true;
   }
+
+  [[gnu::always_inline]]
+  inline __wait_value_type
+  __platform_load(const int* addr, int order, int /* obj_sz */) noexcept
+  { return __atomic_load_n(addr, order); }
+
+#elif defined __FreeBSD__ && __SIZEOF_LONG__ == 8
+  [[gnu::always_inline]]
+  inline int
+  wait_op(int obj_sz) noexcept
+  { return obj_sz == sizeof(unsigned) ? UMTX_OP_WAIT_UINT : UMTX_OP_WAIT; }
+
+  void
+  __platform_wait(const void* addr, uint64_t val, int obj_sz) noexcept
+  {
+    if (_umtx_op(const_cast<void*>(addr), wait_op(obj_sz), val,
+		 nullptr, nullptr))
+      if (errno != EINTR)
+	__throw_system_error(errno);
+  }
+
+  void
+  __platform_notify(const void* addr, bool all, int /* obj_sz */) noexcept
+  {
+    const int count = all ? INT_MAX : 1;
+    _umtx_op(const_cast<void*>(addr), UMTX_OP_WAKE, count, nullptr, nullptr);
+  }
+
+  // returns true if wait ended before timeout
+  bool
+  __platform_wait_until(const void* addr, uint64_t val,
+			const __wait_clock_t::time_point& atime,
+			int obj_sz) noexcept
+  {
+    struct _umtx_time timeout = {
+      ._timeout = chrono::__to_timeout_timespec(atime),
+      ._flags = UMTX_ABSTIME,
+      ._clockid = CLOCK_MONOTONIC
+    };
+    // _umtx_op hangs if timeout._timeout is {0, 0}
+    if (atime.time_since_epoch() < chrono::nanoseconds(1))
+      return false;
+    constexpr uintptr_t timeout_sz = sizeof(timeout);
+    if (_umtx_op(const_cast<void*>(addr), wait_op(obj_sz), val,
+		 (void*)timeout_sz, &timeout))
+      {
+	if (errno == ETIMEDOUT)
+	  return false;
+	if (errno != EINTR)
+	  __throw_system_error(errno);
+      }
+    return true;
+  }
+
+  [[gnu::always_inline]]
+  inline __wait_value_type
+  __platform_load(const void* addr, int order, int obj_sz) noexcept
+  {
+    if (obj_sz == sizeof(long))
+      return __atomic_load_n(static_cast<const long*>(addr), order);
+    return __atomic_load_n(static_cast<const unsigned*>(addr), order);
+  }
 #endif // HAVE_PLATFORM_WAIT
 
   // The state used by atomic waiting and notifying functions.
@@ -259,7 +333,7 @@ namespace
     __wait_value_type wval;
     for (auto i = 0; i < atomic_spin_count; ++i)
       {
-	wval = __atomic_load_n(addr, args._M_order);
+	wval = __platform_load(addr, args._M_order, args._M_obj_size);
 	if (wval != args._M_old)
 	  return { ._M_val = wval, ._M_has_val = true, ._M_timeout = false };
 	if (i < atomic_spin_count_relax)