1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 #ifndef _KERNEL_TIME_MIGRATION_H 3 #define _KERNEL_TIME_MIGRATION_H 4 5 /* Per group capacity. Must be a power of 2! */ 6 #define TMIGR_CHILDREN_PER_GROUP 8 7 8 /** 9 * struct tmigr_event - a timer event associated to a CPU 10 * @nextevt: The node to enqueue an event in the parent group queue 11 * @cpu: The CPU to which this event belongs 12 * @ignore: Hint whether the event could be ignored; it is set when 13 * CPU or group is active; 14 */ 15 struct tmigr_event { 16 struct timerqueue_node nextevt; 17 unsigned int cpu; 18 bool ignore; 19 }; 20 21 /** 22 * struct tmigr_group - timer migration hierarchy group 23 * @lock: Lock protecting the event information and group hierarchy 24 * information during setup 25 * @parent: Pointer to the parent group. Pointer is updated when a 26 * new hierarchy level is added because of a CPU coming 27 * online the first time. Once it is set, the pointer will 28 * not be removed or updated. When accessing parent pointer 29 * lock less to decide whether to abort a propagation or 30 * not, it is not a problem. The worst outcome is an 31 * unnecessary/early CPU wake up. But do not access parent 32 * pointer several times in the same 'action' (like 33 * activation, deactivation, check for remote expiry,...) 34 * without holding the lock as it is not ensured that value 35 * will not change. 36 * @groupevt: Next event of the group which is only used when the 37 * group is !active. The group event is then queued into 38 * the parent timer queue. 39 * Ignore bit of @groupevt is set when the group is active. 40 * @next_expiry: Base monotonic expiry time of the next event of the 41 * group; It is used for the racy lockless check whether a 42 * remote expiry is required; it is always reliable 43 * @events: Timer queue for child events queued in the group 44 * @migr_state: State of the group (see union tmigr_state) 45 * @level: Hierarchy level of the group; Required during setup 46 * @numa_node: Required for setup only to make sure CPU and low level 47 * group information is NUMA local. It is set to NUMA node 48 * as long as the group level is per NUMA node (level < 49 * tmigr_crossnode_level); otherwise it is set to 50 * NUMA_NO_NODE 51 * @num_children: Counter of group children to make sure the group is only 52 * filled with TMIGR_CHILDREN_PER_GROUP; Required for setup 53 * only 54 * @childmask: childmask of the group in the parent group; is set 55 * during setup and will never change; can be read 56 * lockless 57 * @list: List head that is added to the per level 58 * tmigr_level_list; is required during setup when a 59 * new group needs to be connected to the existing 60 * hierarchy groups 61 */ 62 struct tmigr_group { 63 raw_spinlock_t lock; 64 struct tmigr_group *parent; 65 struct tmigr_event groupevt; 66 u64 next_expiry; 67 struct timerqueue_head events; 68 atomic_t migr_state; 69 unsigned int level; 70 int numa_node; 71 unsigned int num_children; 72 u8 childmask; 73 struct list_head list; 74 }; 75 76 /** 77 * struct tmigr_cpu - timer migration per CPU group 78 * @lock: Lock protecting the tmigr_cpu group information 79 * @online: Indicates whether the CPU is online; In deactivate path 80 * it is required to know whether the migrator in the top 81 * level group is to be set offline, while a timer is 82 * pending. Then another online CPU needs to be notified to 83 * take over the migrator role. Furthermore the information 84 * is required in CPU hotplug path as the CPU is able to go 85 * idle before the timer migration hierarchy hotplug AP is 86 * reached. During this phase, the CPU has to handle the 87 * global timers on its own and must not act as a migrator. 88 * @idle: Indicates whether the CPU is idle in the timer migration 89 * hierarchy 90 * @remote: Is set when timers of the CPU are expired remotely 91 * @tmgroup: Pointer to the parent group 92 * @childmask: childmask of tmigr_cpu in the parent group 93 * @wakeup: Stores the first timer when the timer migration 94 * hierarchy is completely idle and remote expiry was done; 95 * is returned to timer code in the idle path and is only 96 * used in idle path. 97 * @cpuevt: CPU event which could be enqueued into the parent group 98 */ 99 struct tmigr_cpu { 100 raw_spinlock_t lock; 101 bool online; 102 bool idle; 103 bool remote; 104 struct tmigr_group *tmgroup; 105 u8 childmask; 106 u64 wakeup; 107 struct tmigr_event cpuevt; 108 }; 109 110 /** 111 * union tmigr_state - state of tmigr_group 112 * @state: Combined version of the state - only used for atomic 113 * read/cmpxchg function 114 * @struct: Split version of the state - only use the struct members to 115 * update information to stay independent of endianness 116 */ 117 union tmigr_state { 118 u32 state; 119 /** 120 * struct - split state of tmigr_group 121 * @active: Contains each childmask bit of the active children 122 * @migrator: Contains childmask of the child which is migrator 123 * @seq: Sequence counter needs to be increased when an update 124 * to the tmigr_state is done. It prevents a race when 125 * updates in the child groups are propagated in changed 126 * order. Detailed information about the scenario is 127 * given in the documentation at the begin of 128 * timer_migration.c. 129 */ 130 struct { 131 u8 active; 132 u8 migrator; 133 u16 seq; 134 } __packed; 135 }; 136 137 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) 138 extern void tmigr_handle_remote(void); 139 extern bool tmigr_requires_handle_remote(void); 140 extern void tmigr_cpu_activate(void); 141 extern u64 tmigr_cpu_deactivate(u64 nextevt); 142 extern u64 tmigr_cpu_new_timer(u64 nextevt); 143 extern u64 tmigr_quick_check(u64 nextevt); 144 #else 145 static inline void tmigr_handle_remote(void) { } 146 static inline bool tmigr_requires_handle_remote(void) { return false; } 147 static inline void tmigr_cpu_activate(void) { } 148 #endif 149 150 #endif 151