1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2018 Intel Corporation 3 */ 4 5 #include <string.h> 6 7 #include <rte_lcore.h> 8 #include <rte_cycles.h> 9 #include <rte_atomic.h> 10 #include <rte_malloc.h> 11 #include <inttypes.h> 12 13 #include "rte_power.h" 14 #include "rte_power_empty_poll.h" 15 16 #define INTERVALS_PER_SECOND 100 /* (10ms) */ 17 #define SECONDS_TO_TRAIN_FOR 2 18 #define DEFAULT_MED_TO_HIGH_PERCENT_THRESHOLD 70 19 #define DEFAULT_HIGH_TO_MED_PERCENT_THRESHOLD 30 20 #define DEFAULT_CYCLES_PER_PACKET 800 21 22 static struct ep_params *ep_params; 23 static uint32_t med_to_high_threshold = DEFAULT_MED_TO_HIGH_PERCENT_THRESHOLD; 24 static uint32_t high_to_med_threshold = DEFAULT_HIGH_TO_MED_PERCENT_THRESHOLD; 25 26 static uint32_t avail_freqs[RTE_MAX_LCORE][NUM_FREQS]; 27 28 static uint32_t total_avail_freqs[RTE_MAX_LCORE]; 29 30 static uint32_t freq_index[NUM_FREQ]; 31 32 static uint32_t 33 get_freq_index(enum freq_val index) 34 { 35 return freq_index[index]; 36 } 37 38 39 static int 40 set_power_freq(int lcore_id, enum freq_val freq, bool specific_freq) 41 { 42 int err = 0; 43 uint32_t power_freq_index; 44 if (!specific_freq) 45 power_freq_index = get_freq_index(freq); 46 else 47 power_freq_index = freq; 48 49 err = rte_power_set_freq(lcore_id, power_freq_index); 50 51 return err; 52 } 53 54 55 static __rte_always_inline void 56 exit_training_state(struct priority_worker *poll_stats) 57 { 58 RTE_SET_USED(poll_stats); 59 } 60 61 static __rte_always_inline void 62 enter_training_state(struct priority_worker *poll_stats) 63 { 64 poll_stats->iter_counter = 0; 65 poll_stats->cur_freq = LOW; 66 poll_stats->queue_state = TRAINING; 67 } 68 69 static __rte_always_inline void 70 enter_normal_state(struct priority_worker *poll_stats) 71 { 72 /* Clear the averages arrays and strs */ 73 memset(poll_stats->edpi_av, 0, sizeof(poll_stats->edpi_av)); 74 poll_stats->ec = 0; 75 76 poll_stats->cur_freq = MED; 77 poll_stats->iter_counter = 0; 78 poll_stats->threshold_ctr = 0; 79 poll_stats->queue_state = MED_NORMAL; 80 RTE_LOG(INFO, POWER, "Set the power freq to MED\n"); 81 set_power_freq(poll_stats->lcore_id, MED, false); 82 83 poll_stats->thresh[MED].threshold_percent = med_to_high_threshold; 84 poll_stats->thresh[HGH].threshold_percent = high_to_med_threshold; 85 } 86 87 static __rte_always_inline void 88 enter_busy_state(struct priority_worker *poll_stats) 89 { 90 memset(poll_stats->edpi_av, 0, sizeof(poll_stats->edpi_av)); 91 poll_stats->ec = 0; 92 93 poll_stats->cur_freq = HGH; 94 poll_stats->iter_counter = 0; 95 poll_stats->threshold_ctr = 0; 96 poll_stats->queue_state = HGH_BUSY; 97 set_power_freq(poll_stats->lcore_id, HGH, false); 98 } 99 100 static __rte_always_inline void 101 enter_purge_state(struct priority_worker *poll_stats) 102 { 103 poll_stats->iter_counter = 0; 104 poll_stats->queue_state = LOW_PURGE; 105 } 106 107 static __rte_always_inline void 108 set_state(struct priority_worker *poll_stats, 109 enum queue_state new_state) 110 { 111 enum queue_state old_state = poll_stats->queue_state; 112 if (old_state != new_state) { 113 114 /* Call any old state exit functions */ 115 if (old_state == TRAINING) 116 exit_training_state(poll_stats); 117 118 /* Call any new state entry functions */ 119 if (new_state == TRAINING) 120 enter_training_state(poll_stats); 121 if (new_state == MED_NORMAL) 122 enter_normal_state(poll_stats); 123 if (new_state == HGH_BUSY) 124 enter_busy_state(poll_stats); 125 if (new_state == LOW_PURGE) 126 enter_purge_state(poll_stats); 127 } 128 } 129 130 static __rte_always_inline void 131 set_policy(struct priority_worker *poll_stats, 132 struct ep_policy *policy) 133 { 134 set_state(poll_stats, policy->state); 135 136 if (policy->state == TRAINING) 137 return; 138 139 poll_stats->thresh[MED_NORMAL].base_edpi = policy->med_base_edpi; 140 poll_stats->thresh[HGH_BUSY].base_edpi = policy->hgh_base_edpi; 141 142 poll_stats->thresh[MED_NORMAL].trained = true; 143 poll_stats->thresh[HGH_BUSY].trained = true; 144 145 } 146 147 static void 148 update_training_stats(struct priority_worker *poll_stats, 149 uint32_t freq, 150 bool specific_freq, 151 uint32_t max_train_iter) 152 { 153 RTE_SET_USED(specific_freq); 154 155 uint64_t p0_empty_deq; 156 157 if (poll_stats->cur_freq == freq && 158 poll_stats->thresh[freq].trained == false) { 159 if (poll_stats->thresh[freq].cur_train_iter == 0) { 160 161 set_power_freq(poll_stats->lcore_id, 162 freq, specific_freq); 163 164 poll_stats->empty_dequeues_prev = 165 poll_stats->empty_dequeues; 166 167 poll_stats->thresh[freq].cur_train_iter++; 168 169 return; 170 } else if (poll_stats->thresh[freq].cur_train_iter 171 <= max_train_iter) { 172 173 p0_empty_deq = poll_stats->empty_dequeues - 174 poll_stats->empty_dequeues_prev; 175 176 poll_stats->empty_dequeues_prev = 177 poll_stats->empty_dequeues; 178 179 poll_stats->thresh[freq].base_edpi += p0_empty_deq; 180 poll_stats->thresh[freq].cur_train_iter++; 181 182 } else { 183 if (poll_stats->thresh[freq].trained == false) { 184 poll_stats->thresh[freq].base_edpi = 185 poll_stats->thresh[freq].base_edpi / 186 max_train_iter; 187 188 /* Add on a factor of 0.05% 189 * this should remove any 190 * false negatives when the system is 0% busy 191 */ 192 poll_stats->thresh[freq].base_edpi += 193 poll_stats->thresh[freq].base_edpi / 2000; 194 195 poll_stats->thresh[freq].trained = true; 196 poll_stats->cur_freq++; 197 198 } 199 } 200 } 201 } 202 203 static __rte_always_inline uint32_t 204 update_stats(struct priority_worker *poll_stats) 205 { 206 uint64_t tot_edpi = 0; 207 uint32_t j, percent; 208 209 struct priority_worker *s = poll_stats; 210 211 uint64_t cur_edpi = s->empty_dequeues - s->empty_dequeues_prev; 212 213 s->empty_dequeues_prev = s->empty_dequeues; 214 215 if (s->thresh[s->cur_freq].base_edpi < cur_edpi) { 216 217 /* edpi mean empty poll counter difference per interval */ 218 RTE_LOG(DEBUG, POWER, "cur_edpi is too large " 219 "cur edpi %"PRId64" " 220 "base edpi %"PRId64"\n", 221 cur_edpi, 222 s->thresh[s->cur_freq].base_edpi); 223 /* Value to make us fail need debug log*/ 224 return 1000UL; 225 } 226 227 s->edpi_av[s->ec++ % BINS_AV] = cur_edpi; 228 229 for (j = 0; j < BINS_AV; j++) { 230 tot_edpi += s->edpi_av[j]; 231 } 232 233 tot_edpi = tot_edpi / BINS_AV; 234 235 percent = 100 - (uint32_t)(((float)tot_edpi / 236 (float)s->thresh[s->cur_freq].base_edpi) * 100); 237 238 return (uint32_t)percent; 239 } 240 241 242 static __rte_always_inline void 243 update_stats_normal(struct priority_worker *poll_stats) 244 { 245 uint32_t percent; 246 247 if (poll_stats->thresh[poll_stats->cur_freq].base_edpi == 0) { 248 249 enum freq_val cur_freq = poll_stats->cur_freq; 250 251 /* edpi mean empty poll counter difference per interval */ 252 RTE_LOG(DEBUG, POWER, "cure freq is %d, edpi is %"PRIu64"\n", 253 cur_freq, 254 poll_stats->thresh[cur_freq].base_edpi); 255 return; 256 } 257 258 percent = update_stats(poll_stats); 259 260 if (percent > 100) { 261 /* edpi mean empty poll counter difference per interval */ 262 RTE_LOG(DEBUG, POWER, "Edpi is bigger than threshold\n"); 263 return; 264 } 265 266 if (poll_stats->cur_freq == LOW) 267 RTE_LOG(INFO, POWER, "Purge Mode is not currently supported\n"); 268 else if (poll_stats->cur_freq == MED) { 269 270 if (percent > 271 poll_stats->thresh[MED].threshold_percent) { 272 273 if (poll_stats->threshold_ctr < INTERVALS_PER_SECOND) 274 poll_stats->threshold_ctr++; 275 else { 276 set_state(poll_stats, HGH_BUSY); 277 RTE_LOG(INFO, POWER, "MOVE to HGH\n"); 278 } 279 280 } else { 281 /* reset */ 282 poll_stats->threshold_ctr = 0; 283 } 284 285 } else if (poll_stats->cur_freq == HGH) { 286 287 if (percent < 288 poll_stats->thresh[HGH].threshold_percent) { 289 290 if (poll_stats->threshold_ctr < INTERVALS_PER_SECOND) 291 poll_stats->threshold_ctr++; 292 else { 293 set_state(poll_stats, MED_NORMAL); 294 RTE_LOG(INFO, POWER, "MOVE to MED\n"); 295 } 296 } else { 297 /* reset */ 298 poll_stats->threshold_ctr = 0; 299 } 300 301 } 302 } 303 304 static int 305 empty_poll_training(struct priority_worker *poll_stats, 306 uint32_t max_train_iter) 307 { 308 309 if (poll_stats->iter_counter < INTERVALS_PER_SECOND) { 310 poll_stats->iter_counter++; 311 return 0; 312 } 313 314 315 update_training_stats(poll_stats, 316 LOW, 317 false, 318 max_train_iter); 319 320 update_training_stats(poll_stats, 321 MED, 322 false, 323 max_train_iter); 324 325 update_training_stats(poll_stats, 326 HGH, 327 false, 328 max_train_iter); 329 330 331 if (poll_stats->thresh[LOW].trained == true 332 && poll_stats->thresh[MED].trained == true 333 && poll_stats->thresh[HGH].trained == true) { 334 335 set_state(poll_stats, MED_NORMAL); 336 337 RTE_LOG(INFO, POWER, "LOW threshold is %"PRIu64"\n", 338 poll_stats->thresh[LOW].base_edpi); 339 340 RTE_LOG(INFO, POWER, "MED threshold is %"PRIu64"\n", 341 poll_stats->thresh[MED].base_edpi); 342 343 344 RTE_LOG(INFO, POWER, "HIGH threshold is %"PRIu64"\n", 345 poll_stats->thresh[HGH].base_edpi); 346 347 RTE_LOG(INFO, POWER, "Training is Complete for %d\n", 348 poll_stats->lcore_id); 349 } 350 351 return 0; 352 } 353 354 void 355 rte_empty_poll_detection(struct rte_timer *tim, void *arg) 356 { 357 358 uint32_t i; 359 360 struct priority_worker *poll_stats; 361 362 RTE_SET_USED(tim); 363 364 RTE_SET_USED(arg); 365 366 for (i = 0; i < NUM_NODES; i++) { 367 368 poll_stats = &(ep_params->wrk_data.wrk_stats[i]); 369 370 if (rte_lcore_is_enabled(poll_stats->lcore_id) == 0) 371 continue; 372 373 switch (poll_stats->queue_state) { 374 case(TRAINING): 375 empty_poll_training(poll_stats, 376 ep_params->max_train_iter); 377 break; 378 379 case(HGH_BUSY): 380 case(MED_NORMAL): 381 update_stats_normal(poll_stats); 382 break; 383 384 case(LOW_PURGE): 385 break; 386 default: 387 break; 388 389 } 390 391 } 392 393 } 394 395 int 396 rte_power_empty_poll_stat_init(struct ep_params **eptr, uint8_t *freq_tlb, 397 struct ep_policy *policy) 398 { 399 uint32_t i; 400 /* Allocate the ep_params structure */ 401 ep_params = rte_zmalloc_socket(NULL, 402 sizeof(struct ep_params), 403 0, 404 rte_socket_id()); 405 406 if (!ep_params) 407 return -1; 408 409 if (freq_tlb == NULL) { 410 freq_index[LOW] = 14; 411 freq_index[MED] = 9; 412 freq_index[HGH] = 1; 413 } else { 414 freq_index[LOW] = freq_tlb[LOW]; 415 freq_index[MED] = freq_tlb[MED]; 416 freq_index[HGH] = freq_tlb[HGH]; 417 } 418 419 RTE_LOG(INFO, POWER, "Initialize the Empty Poll\n"); 420 421 /* Train for pre-defined period */ 422 ep_params->max_train_iter = INTERVALS_PER_SECOND * SECONDS_TO_TRAIN_FOR; 423 424 struct stats_data *w = &ep_params->wrk_data; 425 426 *eptr = ep_params; 427 428 /* initialize all wrk_stats state */ 429 for (i = 0; i < NUM_NODES; i++) { 430 431 if (rte_lcore_is_enabled(i) == 0) 432 continue; 433 /*init the freqs table */ 434 total_avail_freqs[i] = rte_power_freqs(i, 435 avail_freqs[i], 436 NUM_FREQS); 437 438 RTE_LOG(INFO, POWER, "total avail freq is %d , lcoreid %d\n", 439 total_avail_freqs[i], 440 i); 441 442 if (get_freq_index(LOW) > total_avail_freqs[i]) 443 return -1; 444 445 if (rte_get_main_lcore() != i) { 446 w->wrk_stats[i].lcore_id = i; 447 set_policy(&w->wrk_stats[i], policy); 448 } 449 } 450 451 return 0; 452 } 453 454 void 455 rte_power_empty_poll_stat_free(void) 456 { 457 458 RTE_LOG(INFO, POWER, "Close the Empty Poll\n"); 459 460 if (ep_params != NULL) 461 rte_free(ep_params); 462 } 463 464 int 465 rte_power_empty_poll_stat_update(unsigned int lcore_id) 466 { 467 struct priority_worker *poll_stats; 468 469 if (lcore_id >= NUM_NODES) 470 return -1; 471 472 poll_stats = &(ep_params->wrk_data.wrk_stats[lcore_id]); 473 474 if (poll_stats->lcore_id == 0) 475 poll_stats->lcore_id = lcore_id; 476 477 poll_stats->empty_dequeues++; 478 479 return 0; 480 } 481 482 int 483 rte_power_poll_stat_update(unsigned int lcore_id, uint8_t nb_pkt) 484 { 485 486 struct priority_worker *poll_stats; 487 488 if (lcore_id >= NUM_NODES) 489 return -1; 490 491 poll_stats = &(ep_params->wrk_data.wrk_stats[lcore_id]); 492 493 if (poll_stats->lcore_id == 0) 494 poll_stats->lcore_id = lcore_id; 495 496 poll_stats->num_dequeue_pkts += nb_pkt; 497 498 return 0; 499 } 500 501 502 uint64_t 503 rte_power_empty_poll_stat_fetch(unsigned int lcore_id) 504 { 505 struct priority_worker *poll_stats; 506 507 if (lcore_id >= NUM_NODES) 508 return -1; 509 510 poll_stats = &(ep_params->wrk_data.wrk_stats[lcore_id]); 511 512 if (poll_stats->lcore_id == 0) 513 poll_stats->lcore_id = lcore_id; 514 515 return poll_stats->empty_dequeues; 516 } 517 518 uint64_t 519 rte_power_poll_stat_fetch(unsigned int lcore_id) 520 { 521 struct priority_worker *poll_stats; 522 523 if (lcore_id >= NUM_NODES) 524 return -1; 525 526 poll_stats = &(ep_params->wrk_data.wrk_stats[lcore_id]); 527 528 if (poll_stats->lcore_id == 0) 529 poll_stats->lcore_id = lcore_id; 530 531 return poll_stats->num_dequeue_pkts; 532 } 533