1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5
6 #pragma once
7
8 #include <folly/detail/Futex.h>
9 #include <folly/synchronization/ParkingLot.h>
10
11 #include <condition_variable>
12 #include <cstdint>
13
14 namespace folly {
15 namespace detail {
16 namespace atomic_notification {
17 /**
18 * We use Futex<std::atomic> as the alias that has the lowest performance
19 * overhead with respect to atomic notifications. Assert that
20 * atomic_uint_fast_wait_t is the same as Futex<std::atomic>
21 */
22 static_assert(std::is_same<atomic_uint_fast_wait_t, Futex<std::atomic>>{}, "");
23
24 /**
25 * Implementation and specializations for the atomic_wait() family of
26 * functions
27 */
toCvStatus(FutexResult result)28 inline std::cv_status toCvStatus(FutexResult result) {
29 return (result == FutexResult::TIMEDOUT) ? std::cv_status::timeout
30 : std::cv_status::no_timeout;
31 }
toCvStatus(ParkResult result)32 inline std::cv_status toCvStatus(ParkResult result) {
33 return (result == ParkResult::Timeout) ? std::cv_status::timeout
34 : std::cv_status::no_timeout;
35 }
36
37 // ParkingLot instantiation for futex management
38 extern ParkingLot<std::uint32_t> parkingLot;
39
40 template <template <typename...> class Atom, typename... Args>
atomic_wait_impl(const Atom<std::uint32_t,Args...> * atomic,std::uint32_t expected)41 void atomic_wait_impl(
42 const Atom<std::uint32_t, Args...>* atomic,
43 std::uint32_t expected) {
44 futexWait(atomic, expected);
45 return;
46 }
47
48 template <template <typename...> class Atom, typename Integer, typename... Args>
atomic_wait_impl(const Atom<Integer,Args...> * atomic,Integer expected)49 void atomic_wait_impl(const Atom<Integer, Args...>* atomic, Integer expected) {
50 static_assert(!std::is_same<Integer, std::uint32_t>{}, "");
51 parkingLot.park(
52 atomic, -1, [&] { return atomic->load() == expected; }, [] {});
53 }
54
55 template <
56 template <typename...> class Atom,
57 typename... Args,
58 typename Clock,
59 typename Duration>
atomic_wait_until_impl(const Atom<std::uint32_t,Args...> * atomic,std::uint32_t expected,const std::chrono::time_point<Clock,Duration> & deadline)60 std::cv_status atomic_wait_until_impl(
61 const Atom<std::uint32_t, Args...>* atomic,
62 std::uint32_t expected,
63 const std::chrono::time_point<Clock, Duration>& deadline) {
64 return toCvStatus(futexWaitUntil(atomic, expected, deadline));
65 }
66
67 template <
68 template <typename...> class Atom,
69 typename Integer,
70 typename... Args,
71 typename Clock,
72 typename Duration>
atomic_wait_until_impl(const Atom<Integer,Args...> * atomic,Integer expected,const std::chrono::time_point<Clock,Duration> & deadline)73 std::cv_status atomic_wait_until_impl(
74 const Atom<Integer, Args...>* atomic,
75 Integer expected,
76 const std::chrono::time_point<Clock, Duration>& deadline) {
77 static_assert(!std::is_same<Integer, std::uint32_t>{}, "");
78 return toCvStatus(parkingLot.park_until(
79 atomic, -1, [&] { return atomic->load() == expected; }, [] {}, deadline));
80 }
81
82 template <template <typename...> class Atom, typename... Args>
atomic_notify_one_impl(const Atom<std::uint32_t,Args...> * atomic)83 void atomic_notify_one_impl(const Atom<std::uint32_t, Args...>* atomic) {
84 futexWake(atomic, 1);
85 return;
86 }
87
88 template <template <typename...> class Atom, typename Integer, typename... Args>
atomic_notify_one_impl(const Atom<Integer,Args...> * atomic)89 void atomic_notify_one_impl(const Atom<Integer, Args...>* atomic) {
90 static_assert(!std::is_same<Integer, std::uint32_t>{}, "");
91 parkingLot.unpark(atomic, [&](std::uint32_t data) {
92 assert(data == std::numeric_limits<std::uint32_t>::max());
93 return UnparkControl::RemoveBreak;
94 });
95 }
96
97 template <template <typename...> class Atom, typename... Args>
atomic_notify_all_impl(const Atom<std::uint32_t,Args...> * atomic)98 void atomic_notify_all_impl(const Atom<std::uint32_t, Args...>* atomic) {
99 futexWake(atomic);
100 return;
101 }
102
103 template <template <typename...> class Atom, typename Integer, typename... Args>
atomic_notify_all_impl(const Atom<Integer,Args...> * atomic)104 void atomic_notify_all_impl(const Atom<Integer, Args...>* atomic) {
105 static_assert(!std::is_same<Integer, std::uint32_t>{}, "");
106 parkingLot.unpark(atomic, [&](std::uint32_t data) {
107 assert(data == std::numeric_limits<std::uint32_t>::max());
108 return UnparkControl::RemoveContinue;
109 });
110 }
111 } // namespace atomic_notification
112 } // namespace detail
113
114 template <typename Integer>
atomic_wait(const std::atomic<Integer> * atomic,Integer expected)115 void atomic_wait(const std::atomic<Integer>* atomic, Integer expected) {
116 detail::atomic_notification::atomic_wait_impl(atomic, expected);
117 }
118
119 template <typename Integer, typename Clock, typename Duration>
atomic_wait_until(const std::atomic<Integer> * atomic,Integer expected,const std::chrono::time_point<Clock,Duration> & deadline)120 std::cv_status atomic_wait_until(
121 const std::atomic<Integer>* atomic,
122 Integer expected,
123 const std::chrono::time_point<Clock, Duration>& deadline) {
124 return detail::atomic_notification::atomic_wait_until_impl(
125 atomic, expected, deadline);
126 }
127
128 template <typename Integer>
atomic_notify_one(const std::atomic<Integer> * atomic)129 void atomic_notify_one(const std::atomic<Integer>* atomic) {
130 detail::atomic_notification::atomic_notify_one_impl(atomic);
131 }
132
133 template <typename Integer>
atomic_notify_all(const std::atomic<Integer> * atomic)134 void atomic_notify_all(const std::atomic<Integer>* atomic) {
135 detail::atomic_notification::atomic_notify_all_impl(atomic);
136 }
137
138 } // namespace folly
139