1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 // This source code is licensed under both the GPLv2 (found in the 3 // COPYING file in the root directory) and Apache 2.0 License 4 // (found in the LICENSE.Apache file in the root directory). 5 6 #pragma once 7 8 #include <cstdint> 9 #include <type_traits> 10 11 #include <folly/Traits.h> 12 #include <folly/Utility.h> 13 #include <folly/functional/Invoke.h> 14 #include <folly/lang/Launder.h> 15 16 namespace folly { 17 namespace detail { 18 19 /** 20 * InlineFunctionRef is similar to folly::FunctionRef but has the additional 21 * benefit of being able to store the function it was instantiated with inline 22 * in a buffer of the given capacity. Inline storage is only used if the 23 * function object and a pointer (for type-erasure) are small enough to fit in 24 * the templated size. If there is not enough in-situ capacity for the 25 * callable, this just stores a reference to the function object like 26 * FunctionRef. 27 * 28 * This helps give a perf boost in the case where the data gets separated from 29 * the point of invocation. If, for example, at the point of invocation, the 30 * InlineFunctionRef object is not cached, a remote memory/cache read might be 31 * required to invoke the original callable. Customizable inline storage 32 * helps tune storage so we can store a type-erased callable with better 33 * performance and locality. A real-life example of this might be a 34 * folly::FunctionRef with a function pointer. The folly::FunctionRef would 35 * point to the function pointer object in a remote location. This causes a 36 * double-indirection at the point of invocation, and if that memory is dirty, 37 * or not cached, it would cause additional cache misses. On the other hand 38 * with InlineFunctionRef, inline storage would store the value of the 39 * function pointer, avoiding the need to do a remote lookup to fetch the 40 * value of the function pointer. 41 * 42 * To prevent misuse, InlineFunctionRef disallows construction from an lvalue 43 * callable. This is to prevent usage where a user relies on the callable's 44 * state after invocation through InlineFunctionRef. This has the potential 45 * to copy the callable into inline storage when the callable is small, so we 46 * might not use the same function when invoking, but rather a copy of it. 47 * 48 * Also note that InlineFunctionRef will always invoke the const qualified 49 * version of the call operator for any callable that is passed. Regardless 50 * of whether it has a non-const version. This is done to enforce the logical 51 * constraint of function state being immutable. 52 * 53 * This class is always trivially-copyable (and therefore 54 * trivially-destructible), making it suitable for use in a union without 55 * requiring manual destruction. 56 */ 57 template <typename FunctionType, std::size_t Size> 58 class InlineFunctionRef; 59 60 template <typename ReturnType, typename... Args, std::size_t Size> 61 class InlineFunctionRef<ReturnType(Args...), Size> { 62 using Storage = 63 _t<std::aligned_storage<Size - sizeof(uintptr_t), sizeof(uintptr_t)>>; 64 using Call = ReturnType (*)(const Storage&, Args&&...); 65 66 struct InSituTag {}; 67 struct RefTag {}; 68 69 static_assert( 70 (Size % sizeof(uintptr_t)) == 0, 71 "Size has to be a multiple of sizeof(uintptr_t)"); 72 static_assert(Size >= 2 * sizeof(uintptr_t), "This doesn't work"); 73 static_assert(alignof(Call) == alignof(Storage), "Mismatching alignments"); 74 75 // This defines a mode tag that is used in the construction of 76 // InlineFunctionRef to determine the storage and indirection method for the 77 // passed callable. 78 // 79 // This requires that the we pass in a type that is not ref-qualified. 80 template <typename Func> 81 using ConstructMode = _t<std::conditional< 82 folly::is_trivially_copyable<Func>{} && 83 (sizeof(Func) <= sizeof(Storage)) && 84 (alignof(Func) <= alignof(Storage)), 85 InSituTag, 86 RefTag>>; 87 88 public: 89 /** 90 * InlineFunctionRef can be constructed from a nullptr, callable or another 91 * InlineFunctionRef with the same size. These are the constructors that 92 * don't take a callable. 93 * 94 * InlineFunctionRef is meant to be trivially copyable so we default the 95 * constructors and assignment operators. 96 */ InlineFunctionRef(std::nullptr_t)97 InlineFunctionRef(std::nullptr_t) : call_{nullptr} {} InlineFunctionRef()98 InlineFunctionRef() : call_{nullptr} {} 99 InlineFunctionRef(const InlineFunctionRef& other) = default; 100 InlineFunctionRef(InlineFunctionRef&&) = default; 101 InlineFunctionRef& operator=(const InlineFunctionRef&) = default; 102 InlineFunctionRef& operator=(InlineFunctionRef&&) = default; 103 104 /** 105 * Constructors from callables. 106 * 107 * If all of the following conditions are satisfied, then we store the 108 * callable in the inline storage: 109 * 110 * 1) The function has been passed as an rvalue, meaning that there is no 111 * use of the original in the user's code after it has been passed to 112 * us. 113 * 2) Size of the callable is less than the size of the inline storage 114 * buffer. 115 * 3) The callable is trivially constructible and destructible. 116 * 117 * If any one of the above conditions is not satisfied, we fall back to 118 * reference semantics and store the function as a pointer, and add a level 119 * of indirection through type erasure. 120 */ 121 template < 122 typename Func, 123 _t<std::enable_if< 124 !std::is_same<_t<std::decay<Func>>, InlineFunctionRef>{} && 125 !std::is_reference<Func>{} && 126 std::is_convertible< 127 decltype(std::declval<Func&&>()(std::declval<Args&&>()...)), 128 ReturnType>{}>>* = nullptr> InlineFunctionRef(Func && func)129 InlineFunctionRef(Func&& func) { 130 // We disallow construction from lvalues, so assert that this is not a 131 // reference type. When invoked with an lvalue, Func is a lvalue 132 // reference type, when invoked with an rvalue, Func is not ref-qualified. 133 static_assert( 134 !std::is_reference<Func>{}, 135 "InlineFunctionRef cannot be used with lvalues"); 136 static_assert(std::is_rvalue_reference<Func&&>{}, ""); 137 construct(ConstructMode<Func>{}, folly::as_const(func)); 138 } 139 140 /** 141 * The call operator uses the function pointer and a reference to the 142 * storage to do the dispatch. The function pointer takes care of the 143 * appropriate casting. 144 */ operator()145 ReturnType operator()(Args... args) const { 146 return call_(storage_, static_cast<Args&&>(args)...); 147 } 148 149 /** 150 * We have a function engaged if the call function points to anything other 151 * than null. 152 */ 153 operator bool() const noexcept { 154 return call_; 155 } 156 157 private: 158 friend class InlineFunctionRefTest; 159 160 /** 161 * Inline storage constructor implementation. 162 */ 163 template <typename Func> construct(InSituTag,Func & func)164 void construct(InSituTag, Func& func) { 165 using Value = _t<std::remove_reference<Func>>; 166 167 // Assert that the following two assumptions are valid 168 // 1) fit in the storage space we have and match alignments, and 169 // 2) be invocable in a const context, it does not make sense to copy a 170 // callable into inline storage if it makes state local 171 // modifications. 172 static_assert(alignof(Value) <= alignof(Storage), ""); 173 static_assert(is_invocable<const _t<std::decay<Func>>, Args&&...>{}, ""); 174 static_assert(folly::is_trivially_copyable<Value>{}, ""); 175 176 new (&storage_) Value{func}; 177 call_ = &callInline<Value>; 178 } 179 180 /** 181 * Ref storage constructor implementation. This is identical to 182 * folly::FunctionRef. 183 */ 184 template <typename Func> construct(RefTag,Func & func)185 void construct(RefTag, Func& func) { 186 // store a pointer to the function 187 using Pointer = _t<std::add_pointer<_t<std::remove_reference<Func>>>>; 188 new (&storage_) Pointer{&func}; 189 call_ = &callPointer<Pointer>; 190 } 191 192 template <typename Func> callInline(const Storage & object,Args &&...args)193 static ReturnType callInline(const Storage& object, Args&&... args) { 194 // The only type of pointer allowed is a function pointer, no other 195 // pointer types are invocable. 196 static_assert( 197 !std::is_pointer<Func>::value || 198 std::is_function<_t<std::remove_pointer<Func>>>::value, 199 ""); 200 return (*folly::launder(reinterpret_cast<const Func*>(&object)))( 201 static_cast<Args&&>(args)...); 202 } 203 204 template <typename Func> callPointer(const Storage & object,Args &&...args)205 static ReturnType callPointer(const Storage& object, Args&&... args) { 206 // When the function we were instantiated with was not trivial, the given 207 // pointer points to a pointer, which pointers to the callable. So we 208 // cast to a pointer and then to the pointee. 209 static_assert(std::is_pointer<Func>::value, ""); 210 return (**folly::launder(reinterpret_cast<const Func*>(&object)))( 211 static_cast<Args&&>(args)...); 212 } 213 214 Call call_; 215 Storage storage_; 216 }; 217 218 } // namespace detail 219 } // namespace folly 220