1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_PSI_TYPES_H 3 #define _LINUX_PSI_TYPES_H 4 5 #include <linux/kthread.h> 6 #include <linux/seqlock.h> 7 #include <linux/types.h> 8 #include <linux/kref.h> 9 #include <linux/wait.h> 10 11 #ifdef CONFIG_PSI 12 13 /* Tracked task states */ 14 enum psi_task_count { 15 NR_IOWAIT, 16 NR_MEMSTALL, 17 NR_RUNNING, 18 /* 19 * For IO and CPU stalls the presence of running/oncpu tasks 20 * in the domain means a partial rather than a full stall. 21 * For memory it's not so simple because of page reclaimers: 22 * they are running/oncpu while representing a stall. To tell 23 * whether a domain has productivity left or not, we need to 24 * distinguish between regular running (i.e. productive) 25 * threads and memstall ones. 26 */ 27 NR_MEMSTALL_RUNNING, 28 NR_PSI_TASK_COUNTS = 4, 29 }; 30 31 /* Task state bitmasks */ 32 #define TSK_IOWAIT (1 << NR_IOWAIT) 33 #define TSK_MEMSTALL (1 << NR_MEMSTALL) 34 #define TSK_RUNNING (1 << NR_RUNNING) 35 #define TSK_MEMSTALL_RUNNING (1 << NR_MEMSTALL_RUNNING) 36 37 /* Only one task can be scheduled, no corresponding task count */ 38 #define TSK_ONCPU (1 << NR_PSI_TASK_COUNTS) 39 40 /* Resources that workloads could be stalled on */ 41 enum psi_res { 42 PSI_IO, 43 PSI_MEM, 44 PSI_CPU, 45 NR_PSI_RESOURCES = 3, 46 }; 47 48 /* 49 * Pressure states for each resource: 50 * 51 * SOME: Stalled tasks & working tasks 52 * FULL: Stalled tasks & no working tasks 53 */ 54 enum psi_states { 55 PSI_IO_SOME, 56 PSI_IO_FULL, 57 PSI_MEM_SOME, 58 PSI_MEM_FULL, 59 PSI_CPU_SOME, 60 PSI_CPU_FULL, 61 /* Only per-CPU, to weigh the CPU in the global average: */ 62 PSI_NONIDLE, 63 NR_PSI_STATES = 7, 64 }; 65 66 /* Use one bit in the state mask to track TSK_ONCPU */ 67 #define PSI_ONCPU (1 << NR_PSI_STATES) 68 69 enum psi_aggregators { 70 PSI_AVGS = 0, 71 PSI_POLL, 72 NR_PSI_AGGREGATORS, 73 }; 74 75 struct psi_group_cpu { 76 /* 1st cacheline updated by the scheduler */ 77 78 /* Aggregator needs to know of concurrent changes */ 79 seqcount_t seq ____cacheline_aligned_in_smp; 80 81 /* States of the tasks belonging to this group */ 82 unsigned int tasks[NR_PSI_TASK_COUNTS]; 83 84 /* Aggregate pressure state derived from the tasks */ 85 u32 state_mask; 86 87 /* Period time sampling buckets for each state of interest (ns) */ 88 u32 times[NR_PSI_STATES]; 89 90 /* Time of last task change in this group (rq_clock) */ 91 u64 state_start; 92 93 /* 2nd cacheline updated by the aggregator */ 94 95 /* Delta detection against the sampling buckets */ 96 u32 times_prev[NR_PSI_AGGREGATORS][NR_PSI_STATES] 97 ____cacheline_aligned_in_smp; 98 }; 99 100 /* PSI growth tracking window */ 101 struct psi_window { 102 /* Window size in ns */ 103 u64 size; 104 105 /* Start time of the current window in ns */ 106 u64 start_time; 107 108 /* Value at the start of the window */ 109 u64 start_value; 110 111 /* Value growth in the previous window */ 112 u64 prev_growth; 113 }; 114 115 struct psi_trigger { 116 /* PSI state being monitored by the trigger */ 117 enum psi_states state; 118 119 /* User-spacified threshold in ns */ 120 u64 threshold; 121 122 /* List node inside triggers list */ 123 struct list_head node; 124 125 /* Backpointer needed during trigger destruction */ 126 struct psi_group *group; 127 128 /* Wait queue for polling */ 129 wait_queue_head_t event_wait; 130 131 /* Pending event flag */ 132 int event; 133 134 /* Tracking window */ 135 struct psi_window win; 136 137 /* 138 * Time last event was generated. Used for rate-limiting 139 * events to one per window 140 */ 141 u64 last_event_time; 142 143 /* Deferred event(s) from previous ratelimit window */ 144 bool pending_event; 145 }; 146 147 struct psi_group { 148 /* Protects data used by the aggregator */ 149 struct mutex avgs_lock; 150 151 /* Per-cpu task state & time tracking */ 152 struct psi_group_cpu __percpu *pcpu; 153 154 /* Running pressure averages */ 155 u64 avg_total[NR_PSI_STATES - 1]; 156 u64 avg_last_update; 157 u64 avg_next_update; 158 159 /* Aggregator work control */ 160 struct delayed_work avgs_work; 161 162 /* Total stall times and sampled pressure averages */ 163 u64 total[NR_PSI_AGGREGATORS][NR_PSI_STATES - 1]; 164 unsigned long avg[NR_PSI_STATES - 1][3]; 165 166 /* Monitor work control */ 167 struct task_struct __rcu *poll_task; 168 struct timer_list poll_timer; 169 wait_queue_head_t poll_wait; 170 atomic_t poll_wakeup; 171 172 /* Protects data used by the monitor */ 173 struct mutex trigger_lock; 174 175 /* Configured polling triggers */ 176 struct list_head triggers; 177 u32 nr_triggers[NR_PSI_STATES - 1]; 178 u32 poll_states; 179 u64 poll_min_period; 180 181 /* Total stall times at the start of monitor activation */ 182 u64 polling_total[NR_PSI_STATES - 1]; 183 u64 polling_next_update; 184 u64 polling_until; 185 }; 186 187 #else /* CONFIG_PSI */ 188 189 struct psi_group { }; 190 191 #endif /* CONFIG_PSI */ 192 193 #endif /* _LINUX_PSI_TYPES_H */ 194