1 // Copyright (c) 2023 Apple Inc. All rights reserved. 2 3 #include <darwintest.h> 4 #include <darwintest_utils.h> 5 6 #include "sched_test_harness/sched_clutch_harness.h" 7 8 T_GLOBAL_META(T_META_NAMESPACE("xnu.scheduler"), 9 T_META_RADAR_COMPONENT_NAME("xnu"), 10 T_META_RADAR_COMPONENT_VERSION("scheduler"), 11 T_META_RUN_CONCURRENTLY(true), 12 T_META_OWNER("emily_peterson")); 13 14 #define NUM_RAND_SEEDS 5 15 static unsigned int rand_seeds[NUM_RAND_SEEDS] = {377111, 2738572, 1717171, 4990221, 777777}; 16 17 T_DECL(clutch_runq_processor_bound, 18 "Processor-bound threads vs. Regular threads") 19 { 20 int ret; 21 init_harness("processor_bound"); 22 23 struct thread_group *high_tg = create_tg(clutch_interactivity_score_max); 24 struct thread_group *low_tg = create_tg(0); 25 26 test_thread_t lowest_bound = create_thread(TH_BUCKET_SHARE_BG, low_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_BG]); 27 set_thread_processor_bound(lowest_bound); 28 test_thread_t highest_bound = create_thread(TH_BUCKET_SHARE_IN, high_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_IN]); 29 set_thread_processor_bound(highest_bound); 30 test_thread_t lowest_unbound = create_thread(TH_BUCKET_SHARE_BG, low_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_BG]); 31 test_thread_t highest_unbound = create_thread(TH_BUCKET_SHARE_IN, high_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_IN]); 32 33 for (int i = 0; i < NUM_RAND_SEEDS; i++) { 34 enqueue_threads_rand_order(rand_seeds[i], 4, lowest_bound, highest_bound, lowest_unbound, highest_unbound); 35 ret = dequeue_threads_expect_ordered(4, highest_bound, highest_unbound, lowest_bound, lowest_unbound); 36 T_QUIET; T_EXPECT_EQ(ret, -1, "Processor-bound failed to win tie-break"); 37 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 38 } 39 T_PASS("Processor-bound threads win priority tie-breaker"); 40 41 test_thread_t bound = create_thread(TH_BUCKET_SHARE_DF, low_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_DF] - 1); 42 set_thread_processor_bound(bound); 43 test_thread_t higherpri_unbound = create_thread(TH_BUCKET_SHARE_DF, low_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_DF]); 44 test_thread_t interactive_higherpri_unbound = create_thread(TH_BUCKET_SHARE_DF, high_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_DF]); 45 test_thread_t interactive_lowerpri_unbound = create_thread(TH_BUCKET_SHARE_DF, high_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_DF] - 2); 46 for (int i = 0; i < NUM_RAND_SEEDS; i++) { 47 enqueue_threads_rand_order(rand_seeds[i], 4, bound, higherpri_unbound, interactive_higherpri_unbound, interactive_lowerpri_unbound); 48 ret = dequeue_threads_expect_ordered(4, interactive_higherpri_unbound, bound, interactive_lowerpri_unbound, higherpri_unbound); 49 T_QUIET; T_EXPECT_EQ(ret, -1, "Priority and Clutch interactivity score not factored correctly against processor-bound thread"); 50 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 51 } 52 T_PASS("Clutch root represented against processor-bound threads by highest pri thread in the highest pri Clutch bucket"); 53 } 54 55 T_DECL(clutch_runq_aboveui, 56 "Above UI vs. timeshare FG root buckets") 57 { 58 int ret; 59 init_harness("aboveui"); 60 61 struct thread_group *same_tg = create_tg(clutch_interactivity_score_max); 62 test_thread_t aboveui = create_thread(TH_BUCKET_FIXPRI, same_tg, root_bucket_to_highest_pri[TH_BUCKET_FIXPRI]); 63 set_thread_sched_mode(aboveui, TH_MODE_FIXED); 64 test_thread_t low_fg = create_thread(TH_BUCKET_SHARE_FG, same_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_FG]); 65 test_thread_t high_fg = create_thread(TH_BUCKET_SHARE_FG, same_tg, root_bucket_to_highest_pri[TH_BUCKET_FIXPRI] + 1); 66 67 for (int i = 0; i < NUM_RAND_SEEDS; i++) { 68 enqueue_threads_rand_order(rand_seeds[i], 3, aboveui, low_fg, high_fg); 69 ret = dequeue_threads_expect_ordered(3, high_fg, aboveui, low_fg); 70 T_QUIET; T_EXPECT_EQ(ret, -1, "Aboveui vs. foreground threads dequeued out of order"); 71 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 72 } 73 T_PASS("Aboveui vs. foreground ordered according to priority"); 74 } 75 76 T_DECL(clutch_runq_diff_root_bucket, 77 "Different root buckets (EDF, Starvation Avoidance Mode, and Warp)") 78 { 79 int ret; 80 init_harness("diff_root_bucket_edf"); 81 82 struct thread_group *same_tg = create_tg(0); 83 int num_threads = TH_BUCKET_SCHED_MAX - 1; 84 test_thread_t threads[num_threads]; 85 test_thread_t rev_threads[num_threads]; 86 test_thread_t warper_threads[num_threads]; 87 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) { 88 threads[bucket - 1] = create_thread(bucket, same_tg, root_bucket_to_highest_pri[bucket]); 89 rev_threads[num_threads - bucket] = threads[bucket - 1]; 90 warper_threads[bucket - 1] = create_thread(bucket, same_tg, root_bucket_to_highest_pri[bucket]); 91 } 92 93 /* Validate natural EDF between root buckets */ 94 for (int i = 0; i < NUM_RAND_SEEDS; i++) { 95 enqueue_threads_arr_rand_order(rand_seeds[i], num_threads, threads); 96 ret = dequeue_threads_expect_ordered_arr(num_threads, threads); 97 T_QUIET; T_EXPECT_EQ(ret, -1, "Root buckets dequeued out of EDF order, after the first %d threads dequeued were correct", ret); 98 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 99 } 100 T_PASS("Basic EDF root bucket order respected"); 101 102 /* Warp lets high root buckets win despite reverse ordering of root bucket deadlines */ 103 for (int bucket = TH_BUCKET_SHARE_BG; bucket >= TH_BUCKET_SHARE_FG; bucket--) { 104 if (bucket < TH_BUCKET_SHARE_BG) { 105 increment_mock_time_us(clutch_root_bucket_wcel_us[bucket + 1] - clutch_root_bucket_wcel_us[bucket] + 1); 106 } 107 enqueue_thread(warper_threads[bucket - 1]); 108 enqueue_thread(threads[bucket - 1]); 109 } 110 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) { 111 ret = dequeue_thread_expect(warper_threads[bucket - 1]); 112 T_QUIET; T_EXPECT_EQ(ret, true, "Root bucket %d failed to warp ahead", bucket); 113 increment_mock_time_us(clutch_root_bucket_warp_us[bucket] / 2); 114 ret = dequeue_thread_expect(threads[bucket - 1]); 115 T_QUIET; T_EXPECT_EQ(ret, true, "Root bucket %d's warp window failed to stay open", bucket); 116 increment_mock_time_us(clutch_root_bucket_warp_us[bucket] / 2 + 1); 117 } 118 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 119 T_PASS("Warping and Warp Windows respected"); 120 121 /* After Warp is exhausted, Starvation Avoidance Mode kicks in to choose the buckets in EDF order */ 122 for (int bucket = TH_BUCKET_SHARE_BG; bucket >= TH_BUCKET_SHARE_FG; bucket--) { 123 if (bucket < TH_BUCKET_SHARE_BG) { 124 increment_mock_time_us(clutch_root_bucket_wcel_us[bucket + 1] - clutch_root_bucket_wcel_us[bucket] + 1); 125 } 126 enqueue_thread(threads[bucket - 1]); 127 } 128 ret = dequeue_threads_expect_ordered_arr(num_threads, rev_threads); 129 T_QUIET; T_EXPECT_EQ(ret, -1, "Starvation avoidance failed to kick in, after the first %d threads dequeued were correct", ret); 130 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 131 T_PASS("Starvation Avoidance Mode respected"); 132 } 133 134 T_DECL(clutch_runq_diff_clutch_bucket, 135 "Same root bucket, different TGs") 136 { 137 int ret; 138 init_harness("diff_clutch_bucket"); 139 140 int num_tgs = clutch_interactivity_score_max + 1; 141 struct thread_group *tgs[num_tgs]; 142 for (int i = 0; i < num_tgs; i++) { 143 tgs[i] = create_tg(i); 144 } 145 146 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) { 147 test_thread_t threads[num_tgs]; 148 for (int i = 0; i < num_tgs; i++) { 149 threads[i] = create_thread(bucket, tgs[clutch_interactivity_score_max - i], root_bucket_to_highest_pri[bucket]); 150 } 151 152 for (int i = 0; i < NUM_RAND_SEEDS; i++) { 153 enqueue_threads_arr_rand_order(rand_seeds[i], num_tgs, threads); 154 ret = dequeue_threads_expect_ordered_arr(num_tgs, threads); 155 T_QUIET; T_EXPECT_EQ(ret, -1, "Unique interactivity scores dequeued out-of-order, after the first %d threads dequeued were correct", ret); 156 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 157 } 158 } 159 T_PASS("Interactivity scores between Clutch buckets respected"); 160 161 struct thread_group *low_tg = create_tg(clutch_interactivity_score_max / 2); 162 struct thread_group *high_tg = create_tg((clutch_interactivity_score_max / 2) + 2); 163 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) { 164 test_thread_t lowpri_but_interactive = create_thread(bucket, high_tg, root_bucket_to_highest_pri[bucket] - 1); 165 test_thread_t highpri = create_thread(bucket, low_tg, root_bucket_to_highest_pri[bucket]); 166 167 for (int order = 0; order < 2; order++) { 168 enqueue_threads(2, (order == 0 ? lowpri_but_interactive : highpri), (order == 0 ? highpri : lowpri_but_interactive)); 169 ret = dequeue_threads_expect_ordered(2, lowpri_but_interactive, highpri); 170 T_QUIET; T_EXPECT_EQ(ret, -1, "Pri %d and i-score %d dequeued before pri %d and i-score %d, enqueue-order %d", root_bucket_to_highest_pri[bucket] - 1, (clutch_interactivity_score_max / 2) + 2, root_bucket_to_highest_pri[bucket], clutch_interactivity_score_max / 2, order); 171 } 172 173 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 174 } 175 T_PASS("Priority correctly combined with interactivity scores to order Clutch buckets"); 176 177 struct thread_group *first_tg = create_tg(clutch_interactivity_score_max / 2); 178 struct thread_group *second_tg = create_tg(clutch_interactivity_score_max / 2); 179 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) { 180 test_thread_t first = create_thread(bucket, first_tg, root_bucket_to_highest_pri[bucket]); 181 test_thread_t second = create_thread(bucket, second_tg, root_bucket_to_highest_pri[bucket]); 182 enqueue_threads(2, first, second); 183 184 ret = dequeue_threads_expect_ordered(2, first, second); 185 T_QUIET; T_EXPECT_EQ(ret, -1, "FIFO order disrespected for threads in two Clutch buckets of equal priority"); 186 187 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 188 } 189 T_PASS("Clutch bucket FIFO order respected, for Clutch buckets with the same priority"); 190 } 191 192 T_DECL(clutch_runq_diff_priority, 193 "Same root bucket, same TG, different priorities") 194 { 195 int ret; 196 init_harness("diff_priority"); 197 198 struct thread_group *same_tg = create_tg(0); 199 200 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) { 201 test_thread_t lowpri = create_thread(bucket, same_tg, root_bucket_to_highest_pri[bucket] - 1); 202 test_thread_t highpri = create_thread(bucket, same_tg, root_bucket_to_highest_pri[bucket]); 203 204 for (int order = 0; order < 2; order++) { 205 enqueue_threads(2, (order == 0 ? lowpri : highpri), (order == 0 ? highpri : lowpri)); 206 ret = dequeue_threads_expect_ordered(2, highpri, lowpri); 207 T_QUIET; T_EXPECT_EQ(ret, -1, "Pri %d dequeued before pri %d, enqueue-order %d", root_bucket_to_highest_pri[bucket] - 1, root_bucket_to_highest_pri[bucket], order); 208 } 209 210 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 211 } 212 T_PASS("sched_pri order respected, for threads in the same Clutch bucket"); 213 214 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) { 215 test_thread_t first = create_thread(bucket, same_tg, root_bucket_to_highest_pri[bucket]); 216 test_thread_t second = create_thread(bucket, same_tg, root_bucket_to_highest_pri[bucket]); 217 enqueue_threads(2, first, second); 218 219 ret = dequeue_threads_expect_ordered(2, first, second); 220 T_QUIET; T_EXPECT_EQ(ret, -1, "FIFO order disrespected for two threads at pri %d", root_bucket_to_highest_pri[bucket]); 221 222 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 223 } 224 T_PASS("Thread FIFO order respected, for threads in the same Clutch bucket with the same sched_pri"); 225 } 226 227 /* 228 * 64 bits of fourth argument to CLUTCH_THREAD_SELECT expected to 229 * match the following layout, ordered from most to least significant bit: 230 * 231 * (reserved 23) (selection_opened_starvation_avoidance_window 1) 232 * | (starvation_avoidance_window_close 12) | (selection_was_edf 1) 233 * | | | | (traverse mode 3) 234 * v v v v v 235 * r----------------------wc----------sc----------wsbec-----t--v--- 236 * ^ ^ ^ ^ ^ 237 * | | | | (version 4) 238 * (warp_window_close 12) | | (cluster_id 6) 239 * | (selection_was_cluster_bound 1) 240 * (selection_opened_warp_window 1) 241 */ 242 #define CTS_VERSION 1ULL 243 #define TRAVERSE_MODE_REMOVE_CONSIDER_CURRENT (1ULL << 4) 244 #define TRAVERSE_MODE_CHECK_PREEMPT (2ULL << 4) 245 #define SELECTION_WAS_EDF (1ULL << 13) 246 #define SELECTION_OPENED_STARVATION_AVOIDANCE_WINDOW (1ULL << 15) | SELECTION_WAS_EDF 247 #define SELECTION_OPENED_WARP_WINDOW (1ULL << 16) 248 #define WINDOW_MASK(bucket, cluster_bound) ( 1ULL << (bucket + cluster_bound * TH_BUCKET_SCHED_MAX) ) 249 #define STARVATION_AVOIDANCE_WINDOW_CLOSE(bucket, cluster_bound) (WINDOW_MASK(bucket, cluster_bound) << 17) 250 #define WARP_WINDOW_CLOSE(bucket, cluster_bound) (WINDOW_MASK(bucket, cluster_bound) << 29) 251 252 T_DECL(clutch_runq_tracepoint_thread_select, 253 "Validate emitted MACH_SCHED_CLUTCH_THREAD_SELECT tracepoints") 254 { 255 int ret; 256 uint64_t root_bucket_arg; 257 init_harness("tracepoint_thread_select"); 258 disable_auto_current_thread(); 259 260 struct thread_group *same_tg = create_tg(0); 261 int num_threads = TH_BUCKET_SCHED_MAX - 1; 262 test_thread_t threads[num_threads]; 263 test_thread_t rev_threads[num_threads]; 264 test_thread_t warper_threads[num_threads]; 265 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) { 266 threads[bucket - 1] = create_thread(bucket, same_tg, root_bucket_to_highest_pri[bucket]); 267 rev_threads[num_threads - bucket] = threads[bucket - 1]; 268 warper_threads[bucket - 1] = create_thread(bucket, same_tg, root_bucket_to_highest_pri[bucket]); 269 } 270 271 /* Natural EDF */ 272 enqueue_threads_arr(num_threads, threads); 273 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) { 274 ret = dequeue_thread_expect(threads[bucket - 1]); 275 T_QUIET; T_EXPECT_EQ(ret, true, "Root bucket %d failed to warp ahead", bucket); 276 root_bucket_arg = SELECTION_WAS_EDF | CTS_VERSION; 277 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, (bucket - 1) * 2, 0, bucket, root_bucket_arg); 278 T_QUIET; T_EXPECT_EQ(ret, true, "EDF CLUTCH_THREAD_SELECT tracepoint"); 279 } 280 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 281 T_PASS("Correct CLUTCH_THREAD_SELECT tracepoint info for EDF selections\n"); 282 283 /* Warp windows */ 284 for (int bucket = TH_BUCKET_SHARE_BG; bucket >= TH_BUCKET_SHARE_FG; bucket--) { 285 if (bucket < TH_BUCKET_SHARE_BG) { 286 increment_mock_time_us(clutch_root_bucket_wcel_us[bucket + 1] - clutch_root_bucket_wcel_us[bucket] + 1); 287 } 288 enqueue_thread(warper_threads[bucket - 1]); 289 enqueue_thread(threads[bucket - 1]); 290 } 291 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) { 292 /* Opens a new warp window */ 293 ret = dequeue_thread_expect(warper_threads[bucket - 1]); 294 T_QUIET; T_EXPECT_EQ(ret, true, "Root bucket %d failed to warp ahead", bucket); 295 root_bucket_arg = (bucket < TH_BUCKET_SHARE_BG ? SELECTION_OPENED_WARP_WINDOW : SELECTION_WAS_EDF) | CTS_VERSION; 296 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, bucket * 2 - 1, 0, bucket, root_bucket_arg); 297 T_QUIET; T_EXPECT_EQ(ret, true, "Open warp window CLUTCH_THREAD_SELECT tracepoint"); 298 299 /* Makes use of the opened warp window */ 300 increment_mock_time_us(clutch_root_bucket_warp_us[bucket] / 2); 301 ret = dequeue_thread_expect(threads[bucket - 1]); 302 T_QUIET; T_EXPECT_EQ(ret, true, "Root bucket %d's warp window failed to stay open", bucket); 303 root_bucket_arg = (bucket < TH_BUCKET_SHARE_BG ? 0 : SELECTION_WAS_EDF) | CTS_VERSION; 304 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, bucket * 2 - 2, 0, bucket, root_bucket_arg); 305 T_QUIET; T_EXPECT_EQ(ret, true, "Active warp window CLUTCH_THREAD_SELECT tracepoint"); 306 307 increment_mock_time_us(clutch_root_bucket_warp_us[bucket] / 2 + 1); 308 } 309 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 310 T_PASS("Correct CLUTCH_THREAD_SELECT tracepoint info for warp windows"); 311 312 /* Starvation avoidance windows */ 313 for (int bucket = TH_BUCKET_SHARE_BG; bucket >= TH_BUCKET_SHARE_FG; bucket--) { 314 if (bucket < TH_BUCKET_SHARE_BG) { 315 increment_mock_time_us(clutch_root_bucket_wcel_us[bucket + 1] - clutch_root_bucket_wcel_us[bucket] + 1); 316 } 317 enqueue_thread(threads[bucket - 1]); 318 } 319 for (int bucket = TH_BUCKET_SHARE_BG; bucket >= TH_BUCKET_SHARE_FG; bucket--) { 320 ret = dequeue_thread_expect(threads[bucket - 1]); 321 T_QUIET; T_EXPECT_EQ(ret, true, "Starvation avoidance failed to kick in for bucket %d", bucket); 322 root_bucket_arg = SELECTION_WAS_EDF | CTS_VERSION; 323 if (bucket == TH_BUCKET_SHARE_BG) { 324 /* Enough time has passed for the warp windows opened in the last phase to be closed in one go */ 325 for (int warping_bucket = TH_BUCKET_SHARE_FG; warping_bucket < TH_BUCKET_SHARE_BG; warping_bucket++) { 326 root_bucket_arg |= WARP_WINDOW_CLOSE(warping_bucket, false); 327 } 328 } 329 if (bucket > TH_BUCKET_SHARE_FG) { 330 root_bucket_arg |= SELECTION_OPENED_STARVATION_AVOIDANCE_WINDOW; 331 } 332 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, (bucket - 1) * 2, 0, bucket, root_bucket_arg); 333 T_QUIET; T_EXPECT_EQ(ret, true, "Open starvation avoidance window CLUTCH_THREAD_SELECT tracepoint"); 334 } 335 increment_mock_time_us(clutch_root_bucket_wcel_us[TH_BUCKET_SHARE_BG]); 336 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) { 337 enqueue_thread(threads[bucket - 1]); 338 } 339 for (int bucket = TH_BUCKET_SHARE_FG; bucket < TH_BUCKET_SCHED_MAX; bucket++) { 340 ret = dequeue_thread_expect(threads[bucket - 1]); 341 T_QUIET; T_EXPECT_EQ(ret, true, "EDF dequeue for bucket %d", bucket); 342 root_bucket_arg = SELECTION_WAS_EDF | CTS_VERSION; 343 if (bucket == TH_BUCKET_SHARE_FG) { 344 /* Enough time has passed for the starvation avoidance windows opened in the last phase to be closed in one go */ 345 for (int starved_bucket = TH_BUCKET_SHARE_BG; starved_bucket > TH_BUCKET_SHARE_FG; starved_bucket--) { 346 root_bucket_arg |= STARVATION_AVOIDANCE_WINDOW_CLOSE(starved_bucket, false); 347 } 348 } 349 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, (bucket - 1) * 2, 0, bucket, root_bucket_arg); 350 T_QUIET; T_EXPECT_EQ(ret, true, "Closing starvation avoidance window or EDF CLUTCH_THREAD_SELECT tracepoint"); 351 } 352 T_QUIET; T_ASSERT_EQ(runqueue_empty(), true, "runqueue_empty"); 353 T_PASS("Correct CLUTCH_THREAD_SELECT tracepoint info for starvation avoidance windows"); 354 355 /* Different runq traverse modes */ 356 set_thread_current(threads[0]); 357 enqueue_thread(threads[1]); 358 ret = dequeue_thread_expect_compare_current(threads[0]); 359 T_QUIET; T_EXPECT_EQ(ret, true, "EDF dequeue current thread for bucket"); 360 root_bucket_arg = TRAVERSE_MODE_REMOVE_CONSIDER_CURRENT | SELECTION_WAS_EDF | CTS_VERSION; 361 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, 0, 0, TH_BUCKET_SHARE_FG, root_bucket_arg); 362 T_QUIET; T_EXPECT_EQ(ret, true, "Current thread EDF CLUTCH_THREAD_SELECT tracepoint"); 363 ret = check_preempt_current(false); 364 T_QUIET; T_EXPECT_EQ(ret, true, "Current thread check preempt"); 365 root_bucket_arg = TRAVERSE_MODE_CHECK_PREEMPT | SELECTION_WAS_EDF | CTS_VERSION; 366 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, 0, 0, TH_BUCKET_SHARE_FG, root_bucket_arg); 367 T_QUIET; T_EXPECT_EQ(ret, true, "Current thread check preempt CLUTCH_THREAD_SELECT tracepoint"); 368 T_PASS("Correct CLUTCH_THREAD_SELECT tracepoint info for current thread (traverse modes)"); 369 } 370 371 T_DECL(clutch_runq_root_bucket_expired_windows, 372 "Root bucket warp and starvation avoidance windows should expire at the right time") 373 { 374 int ret; 375 uint64_t root_bucket_arg; 376 init_harness("root_bucket_expired_windows"); 377 disable_auto_current_thread(); 378 379 struct thread_group *same_tg = create_tg(0); 380 test_thread_t def_thread = create_thread(TH_BUCKET_SHARE_DF, same_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_DF]); 381 test_thread_t in_thread = create_thread(TH_BUCKET_SHARE_IN, same_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_IN]); 382 383 /* Expect user_initiated bucket to warp ahread of starved default bucket */ 384 enqueue_thread(def_thread); 385 increment_mock_time_us(clutch_root_bucket_wcel_us[TH_BUCKET_SHARE_DF] + 1); 386 enqueue_thread(in_thread); 387 ret = dequeue_thread_expect(in_thread); 388 T_QUIET; T_EXPECT_EQ(ret, true, "unexpected bucket"); 389 root_bucket_arg = SELECTION_OPENED_WARP_WINDOW | CTS_VERSION; 390 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, 1, 0, TH_BUCKET_SHARE_IN, root_bucket_arg); 391 T_EXPECT_EQ(ret, true, "IN warped ahead, tracepoint"); 392 393 /* Expect warp window to close and default starvation avoidance window to begin */ 394 enqueue_thread(in_thread); 395 increment_mock_time_us(clutch_root_bucket_warp_us[TH_BUCKET_SHARE_IN] + 1); 396 ret = dequeue_thread_expect(def_thread); 397 T_QUIET; T_EXPECT_EQ(ret, true, "unexpected bucket"); 398 root_bucket_arg = WARP_WINDOW_CLOSE(TH_BUCKET_SHARE_IN, false) | SELECTION_OPENED_STARVATION_AVOIDANCE_WINDOW | CTS_VERSION; 399 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, 0, 0, TH_BUCKET_SHARE_DF, root_bucket_arg); 400 T_EXPECT_EQ(ret, true, "IN closed warp and DEF opened starvation avoidance, tracepoint"); 401 402 /* Expect default starvation avoidance window to close and refresh warp for user_initiated with natural EDF */ 403 enqueue_thread(def_thread); 404 increment_mock_time_us(clutch_root_bucket_wcel_us[TH_BUCKET_SHARE_DF] + 1); 405 ret = dequeue_thread_expect(in_thread); 406 T_QUIET; T_EXPECT_EQ(ret, true, "unexpected bucket"); 407 root_bucket_arg = STARVATION_AVOIDANCE_WINDOW_CLOSE(TH_BUCKET_SHARE_DF, false) | SELECTION_WAS_EDF | CTS_VERSION; 408 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, 1, 0, TH_BUCKET_SHARE_IN, root_bucket_arg); 409 T_EXPECT_EQ(ret, true, "DEF closed starvation avoidance window and IN refreshed warp, tracepoint"); 410 411 /* Expect foreground to warp ahead of starved default bucket */ 412 increment_mock_time_us(clutch_root_bucket_wcel_us[TH_BUCKET_SHARE_DF] + 1); 413 test_thread_t fg_thread = create_thread(TH_BUCKET_SHARE_FG, same_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_FG]); 414 enqueue_thread(fg_thread); 415 ret = dequeue_thread_expect(fg_thread); 416 T_QUIET; T_EXPECT_EQ(ret, true, "unexpected bucket"); 417 root_bucket_arg = SELECTION_OPENED_WARP_WINDOW | CTS_VERSION; 418 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, 2, 0, TH_BUCKET_SHARE_FG, root_bucket_arg); 419 T_EXPECT_EQ(ret, true, "FG opened warp window, tracepoint"); 420 421 /* Expect foreground to close warp window and default to open starvation avoidance window */ 422 increment_mock_time_us(clutch_root_bucket_warp_us[TH_BUCKET_SHARE_FG] + 1); 423 enqueue_thread(fg_thread); 424 ret = dequeue_thread_expect(def_thread); 425 T_QUIET; T_EXPECT_EQ(ret, true, "unexpected bucket"); 426 root_bucket_arg = WARP_WINDOW_CLOSE(TH_BUCKET_SHARE_FG, false) | SELECTION_OPENED_STARVATION_AVOIDANCE_WINDOW | CTS_VERSION; 427 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, 0, 0, TH_BUCKET_SHARE_DF, root_bucket_arg); 428 T_EXPECT_EQ(ret, true, "FG closed warp window and DEF opened starvation avoidance window, tracepoint"); 429 430 /* Expect default to close starvation avoidance window */ 431 increment_mock_time_us(clutch_root_bucket_wcel_us[TH_BUCKET_SHARE_DF] + 1); 432 enqueue_thread(def_thread); 433 ret = dequeue_thread_expect(fg_thread); 434 T_QUIET; T_EXPECT_EQ(ret, true, "unexpected bucket"); 435 root_bucket_arg = STARVATION_AVOIDANCE_WINDOW_CLOSE(TH_BUCKET_SHARE_DF, false) | SELECTION_WAS_EDF | CTS_VERSION; 436 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, 2, 0, TH_BUCKET_SHARE_FG, root_bucket_arg); 437 T_EXPECT_EQ(ret, true, "DEF closed starvation avoidance window and FG refreshed warp, tracepoint"); 438 439 /* 440 * Expect user_initiated to experience a full-length warp window 441 * (none spent on expired default starvation avoidance window rdar://120562509) 442 */ 443 increment_mock_time_us(clutch_root_bucket_wcel_us[TH_BUCKET_SHARE_DF] + 1); 444 enqueue_thread(in_thread); 445 ret = dequeue_thread_expect(in_thread); 446 T_QUIET; T_EXPECT_EQ(ret, true, "unexpected bucket"); 447 root_bucket_arg = SELECTION_OPENED_WARP_WINDOW | CTS_VERSION; 448 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, 1, 0, TH_BUCKET_SHARE_IN, root_bucket_arg); 449 T_EXPECT_EQ(ret, true, "IN opened warp window, tracepoint"); 450 enqueue_thread(in_thread); 451 increment_mock_time_us(clutch_root_bucket_warp_us[TH_BUCKET_SHARE_IN] - 1); 452 ret = dequeue_thread_expect(in_thread); 453 T_QUIET; T_EXPECT_EQ(ret, true, "unexpected bucket"); 454 root_bucket_arg = CTS_VERSION; 455 ret = tracepoint_expect(CLUTCH_THREAD_SELECT, 1, 0, TH_BUCKET_SHARE_IN, root_bucket_arg); 456 T_EXPECT_EQ(ret, true, "IN had full-length warp window, tracepoint"); 457 } 458 459 T_DECL(clutch_runq_interactivity_starts_maxed, 460 "A new Clutch bucket group should start with max interactivity score") 461 { 462 int ret; 463 init_harness("interactivity_starts_maxed"); 464 465 struct thread_group *non_interactive_tg = create_tg(clutch_interactivity_score_max - 1); 466 test_thread_t non_interactive_tg_thread = create_thread(TH_BUCKET_SHARE_DF, non_interactive_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_DF]); 467 enqueue_thread(non_interactive_tg_thread); 468 469 struct thread_group *new_tg = create_tg(-1); 470 test_thread_t new_tg_thread = create_thread(TH_BUCKET_SHARE_DF, new_tg, root_bucket_to_highest_pri[TH_BUCKET_SHARE_DF]); 471 enqueue_thread(new_tg_thread); 472 473 ret = dequeue_thread_expect(new_tg_thread); 474 T_EXPECT_EQ(ret, true, "New TG Clutch bucket is interactive"); 475 476 ret = dequeue_thread_expect(non_interactive_tg_thread); 477 T_EXPECT_EQ(ret, true, "Non-interactive thread comes second"); 478 } 479