1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2016 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <getopt.h> 35 #include <inttypes.h> 36 #include <limits.h> 37 #include <sched.h> 38 #include <signal.h> 39 #include <stdio.h> 40 41 #include <rte_common.h> 42 #include <rte_memcpy.h> 43 44 #include <pqos.h> 45 46 #include "cat.h" 47 48 #define BITS_PER_HEX 4 49 #define PQOS_MAX_SOCKETS 8 50 #define PQOS_MAX_SOCKET_CORES 64 51 #define PQOS_MAX_CORES (PQOS_MAX_SOCKET_CORES * PQOS_MAX_SOCKETS) 52 53 static const struct pqos_cap *m_cap; 54 static const struct pqos_cpuinfo *m_cpu; 55 static const struct pqos_capability *m_cap_l3ca; 56 #if PQOS_VERSION <= 103 57 static unsigned m_sockets[PQOS_MAX_SOCKETS]; 58 #else 59 static unsigned int *m_sockets; 60 #endif 61 static unsigned m_sock_count; 62 static struct cat_config m_config[PQOS_MAX_CORES]; 63 static unsigned m_config_count; 64 65 static unsigned 66 bits_count(uint64_t bitmask) 67 { 68 unsigned count = 0; 69 70 for (; bitmask != 0; count++) 71 bitmask &= bitmask - 1; 72 73 return count; 74 } 75 76 /* 77 * Parse elem, the elem could be single number/range or '(' ')' group 78 * 1) A single number elem, it's just a simple digit. e.g. 9 79 * 2) A single range elem, two digits with a '-' between. e.g. 2-6 80 * 3) A group elem, combines multiple 1) or 2) with '( )'. e.g (0,2-4,6) 81 * Within group elem, '-' used for a range separator; 82 * ',' used for a single number. 83 */ 84 static int 85 parse_set(const char *input, rte_cpuset_t *cpusetp) 86 { 87 unsigned idx; 88 const char *str = input; 89 char *end = NULL; 90 unsigned min, max; 91 const unsigned num = PQOS_MAX_CORES; 92 93 CPU_ZERO(cpusetp); 94 95 while (isblank(*str)) 96 str++; 97 98 /* only digit or left bracket is qualify for start point */ 99 if ((!isdigit(*str) && *str != '(') || *str == '\0') 100 return -1; 101 102 /* process single number or single range of number */ 103 if (*str != '(') { 104 errno = 0; 105 idx = strtoul(str, &end, 10); 106 107 if (errno || end == NULL || idx >= num) 108 return -1; 109 110 while (isblank(*end)) 111 end++; 112 113 min = idx; 114 max = idx; 115 if (*end == '-') { 116 /* process single <number>-<number> */ 117 end++; 118 while (isblank(*end)) 119 end++; 120 if (!isdigit(*end)) 121 return -1; 122 123 errno = 0; 124 idx = strtoul(end, &end, 10); 125 if (errno || end == NULL || idx >= num) 126 return -1; 127 max = idx; 128 while (isblank(*end)) 129 end++; 130 if (*end != ',' && *end != '\0') 131 return -1; 132 } 133 134 if (*end != ',' && *end != '\0' && *end != '@') 135 return -1; 136 137 for (idx = RTE_MIN(min, max); idx <= RTE_MAX(min, max); 138 idx++) 139 CPU_SET(idx, cpusetp); 140 141 return end - input; 142 } 143 144 /* process set within bracket */ 145 str++; 146 while (isblank(*str)) 147 str++; 148 if (*str == '\0') 149 return -1; 150 151 min = PQOS_MAX_CORES; 152 do { 153 154 /* go ahead to the first digit */ 155 while (isblank(*str)) 156 str++; 157 if (!isdigit(*str)) 158 return -1; 159 160 /* get the digit value */ 161 errno = 0; 162 idx = strtoul(str, &end, 10); 163 if (errno || end == NULL || idx >= num) 164 return -1; 165 166 /* go ahead to separator '-',',' and ')' */ 167 while (isblank(*end)) 168 end++; 169 if (*end == '-') { 170 if (min == PQOS_MAX_CORES) 171 min = idx; 172 else /* avoid continuous '-' */ 173 return -1; 174 } else if ((*end == ',') || (*end == ')')) { 175 max = idx; 176 if (min == PQOS_MAX_CORES) 177 min = idx; 178 for (idx = RTE_MIN(min, max); idx <= RTE_MAX(min, max); 179 idx++) 180 CPU_SET(idx, cpusetp); 181 182 min = PQOS_MAX_CORES; 183 } else 184 return -1; 185 186 str = end + 1; 187 } while (*end != '\0' && *end != ')'); 188 189 return str - input; 190 } 191 192 /* Test if bitmask is contiguous */ 193 static int 194 is_contiguous(uint64_t bitmask) 195 { 196 /* check if bitmask is contiguous */ 197 unsigned i = 0; 198 unsigned j = 0; 199 const unsigned max_idx = (sizeof(bitmask) * CHAR_BIT); 200 201 if (bitmask == 0) 202 return 0; 203 204 for (i = 0; i < max_idx; i++) { 205 if (((1ULL << i) & bitmask) != 0) 206 j++; 207 else if (j > 0) 208 break; 209 } 210 211 if (bits_count(bitmask) != j) { 212 printf("PQOS: mask 0x%llx is not contiguous.\n", 213 (unsigned long long)bitmask); 214 return 0; 215 } 216 217 return 1; 218 } 219 220 /* 221 * The format pattern: --l3ca='<cbm@cpus>[,<(ccbm,dcbm)@cpus>...]' 222 * cbm could be a single mask or for a CDP enabled system, a group of two masks 223 * ("code cbm" and "data cbm") 224 * '(' and ')' are necessary if it's a group. 225 * cpus could be a single digit/range or a group. 226 * '(' and ')' are necessary if it's a group. 227 * 228 * e.g. '0x00F00@(1,3), 0x0FF00@(4-6), 0xF0000@7' 229 * - CPUs 1 and 3 share its 4 ways with CPUs 4, 5 and 6; 230 * - CPUs 4,5 and 6 share half (4 out of 8 ways) of its L3 with 1 and 3; 231 * - CPUs 4,5 and 6 have exclusive access to 4 out of 8 ways; 232 * - CPU 7 has exclusive access to all of its 4 ways; 233 * 234 * e.g. '(0x00C00,0x00300)@(1,3)' for a CDP enabled system 235 * - cpus 1 and 3 have access to 2 ways for code and 2 ways for data, 236 * code and data ways are not overlapping.; 237 */ 238 static int 239 parse_l3ca(const char *l3ca) 240 { 241 unsigned idx = 0; 242 const char *cbm_start = NULL; 243 char *cbm_end = NULL; 244 const char *end = NULL; 245 int offset; 246 rte_cpuset_t cpuset; 247 uint64_t mask = 0; 248 uint64_t cmask = 0; 249 250 if (l3ca == NULL) 251 goto err; 252 253 /* Get cbm */ 254 do { 255 CPU_ZERO(&cpuset); 256 mask = 0; 257 cmask = 0; 258 259 while (isblank(*l3ca)) 260 l3ca++; 261 262 if (*l3ca == '\0') 263 goto err; 264 265 /* record mask_set start point */ 266 cbm_start = l3ca; 267 268 /* go across a complete bracket */ 269 if (*cbm_start == '(') { 270 l3ca += strcspn(l3ca, ")"); 271 if (*l3ca++ == '\0') 272 goto err; 273 } 274 275 /* scan the separator '@', ','(next) or '\0'(finish) */ 276 l3ca += strcspn(l3ca, "@,"); 277 278 if (*l3ca != '@') 279 goto err; 280 281 /* explicit assign cpu_set */ 282 offset = parse_set(l3ca + 1, &cpuset); 283 if (offset < 0 || CPU_COUNT(&cpuset) == 0) 284 goto err; 285 286 end = l3ca + 1 + offset; 287 288 if (*end != ',' && *end != '\0') 289 goto err; 290 291 /* parse mask_set from start point */ 292 if (*cbm_start == '(') { 293 cbm_start++; 294 295 while (isblank(*cbm_start)) 296 cbm_start++; 297 298 if (!isxdigit(*cbm_start)) 299 goto err; 300 301 errno = 0; 302 cmask = strtoul(cbm_start, &cbm_end, 16); 303 if (errno != 0 || cbm_end == NULL || cmask == 0) 304 goto err; 305 306 while (isblank(*cbm_end)) 307 cbm_end++; 308 309 if (*cbm_end != ',') 310 goto err; 311 312 cbm_end++; 313 314 while (isblank(*cbm_end)) 315 cbm_end++; 316 317 if (!isxdigit(*cbm_end)) 318 goto err; 319 320 errno = 0; 321 mask = strtoul(cbm_end, &cbm_end, 16); 322 if (errno != 0 || cbm_end == NULL || mask == 0) 323 goto err; 324 } else { 325 while (isblank(*cbm_start)) 326 cbm_start++; 327 328 if (!isxdigit(*cbm_start)) 329 goto err; 330 331 errno = 0; 332 mask = strtoul(cbm_start, &cbm_end, 16); 333 if (errno != 0 || cbm_end == NULL || mask == 0) 334 goto err; 335 336 } 337 338 if (mask == 0 || is_contiguous(mask) == 0) 339 goto err; 340 341 if (cmask != 0 && is_contiguous(cmask) == 0) 342 goto err; 343 344 rte_memcpy(&m_config[idx].cpumask, 345 &cpuset, sizeof(rte_cpuset_t)); 346 347 if (cmask != 0) { 348 m_config[idx].cdp = 1; 349 m_config[idx].code_mask = cmask; 350 m_config[idx].data_mask = mask; 351 } else 352 m_config[idx].mask = mask; 353 354 m_config_count++; 355 356 l3ca = end + 1; 357 idx++; 358 } while (*end != '\0' && idx < PQOS_MAX_CORES); 359 360 return 0; 361 362 err: 363 return -EINVAL; 364 } 365 366 static int 367 check_cpus_overlapping(void) 368 { 369 unsigned i = 0; 370 unsigned j = 0; 371 rte_cpuset_t mask; 372 373 CPU_ZERO(&mask); 374 375 for (i = 0; i < m_config_count; i++) { 376 for (j = i + 1; j < m_config_count; j++) { 377 CPU_AND(&mask, 378 &m_config[i].cpumask, 379 &m_config[j].cpumask); 380 381 if (CPU_COUNT(&mask) != 0) { 382 printf("PQOS: Requested CPUs sets are " 383 "overlapping.\n"); 384 return -EINVAL; 385 } 386 } 387 } 388 389 return 0; 390 } 391 392 static int 393 check_cpus(void) 394 { 395 unsigned i = 0; 396 unsigned cpu_id = 0; 397 unsigned cos_id = 0; 398 int ret = 0; 399 400 for (i = 0; i < m_config_count; i++) { 401 for (cpu_id = 0; cpu_id < PQOS_MAX_CORES; cpu_id++) { 402 if (CPU_ISSET(cpu_id, &m_config[i].cpumask) != 0) { 403 404 ret = pqos_cpu_check_core(m_cpu, cpu_id); 405 if (ret != PQOS_RETVAL_OK) { 406 printf("PQOS: %u is not a valid " 407 "logical core id.\n", cpu_id); 408 ret = -ENODEV; 409 goto exit; 410 } 411 412 #if PQOS_VERSION <= 103 413 ret = pqos_l3ca_assoc_get(cpu_id, &cos_id); 414 #else 415 ret = pqos_alloc_assoc_get(cpu_id, &cos_id); 416 #endif 417 if (ret != PQOS_RETVAL_OK) { 418 printf("PQOS: Failed to read COS " 419 "associated to cpu %u.\n", 420 cpu_id); 421 ret = -EFAULT; 422 goto exit; 423 } 424 425 /* 426 * Check if COS assigned to lcore is different 427 * then default one (#0) 428 */ 429 if (cos_id != 0) { 430 printf("PQOS: cpu %u has already " 431 "associated COS#%u. " 432 "Please reset L3CA.\n", 433 cpu_id, cos_id); 434 ret = -EBUSY; 435 goto exit; 436 } 437 } 438 } 439 } 440 441 exit: 442 return ret; 443 } 444 445 static int 446 check_cdp(void) 447 { 448 unsigned i = 0; 449 450 for (i = 0; i < m_config_count; i++) { 451 if (m_config[i].cdp == 1 && m_cap_l3ca->u.l3ca->cdp_on == 0) { 452 if (m_cap_l3ca->u.l3ca->cdp == 0) { 453 printf("PQOS: CDP requested but not " 454 "supported.\n"); 455 } else { 456 printf("PQOS: CDP requested but not enabled. " 457 "Please enable CDP.\n"); 458 } 459 return -ENOTSUP; 460 } 461 } 462 463 return 0; 464 } 465 466 static int 467 check_cbm_len_and_contention(void) 468 { 469 unsigned i = 0; 470 uint64_t mask = 0; 471 const uint64_t not_cbm = (UINT64_MAX << (m_cap_l3ca->u.l3ca->num_ways)); 472 const uint64_t cbm_contention_mask = m_cap_l3ca->u.l3ca->way_contention; 473 int ret = 0; 474 475 for (i = 0; i < m_config_count; i++) { 476 if (m_config[i].cdp == 1) 477 mask = m_config[i].code_mask | m_config[i].data_mask; 478 else 479 mask = m_config[i].mask; 480 481 if ((mask & not_cbm) != 0) { 482 printf("PQOS: One or more of requested CBM masks not " 483 "supported by system (too long).\n"); 484 ret = -ENOTSUP; 485 break; 486 } 487 488 /* Just a warning */ 489 if ((mask & cbm_contention_mask) != 0) { 490 printf("PQOS: One or more of requested CBM masks " 491 "overlap CBM contention mask.\n"); 492 break; 493 } 494 495 } 496 497 return ret; 498 } 499 500 static int 501 check_and_select_classes(unsigned cos_id_map[][PQOS_MAX_SOCKETS]) 502 { 503 unsigned i = 0; 504 unsigned j = 0; 505 unsigned phy_pkg_id = 0; 506 unsigned cos_id = 0; 507 unsigned cpu_id = 0; 508 unsigned phy_pkg_lcores[PQOS_MAX_SOCKETS][m_config_count]; 509 const unsigned cos_num = m_cap_l3ca->u.l3ca->num_classes; 510 unsigned used_cos_table[PQOS_MAX_SOCKETS][cos_num]; 511 int ret = 0; 512 513 memset(phy_pkg_lcores, 0, sizeof(phy_pkg_lcores)); 514 memset(used_cos_table, 0, sizeof(used_cos_table)); 515 516 /* detect currently used COS */ 517 for (j = 0; j < m_cpu->num_cores; j++) { 518 cpu_id = m_cpu->cores[j].lcore; 519 520 #if PQOS_VERSION <= 103 521 ret = pqos_l3ca_assoc_get(cpu_id, &cos_id); 522 #else 523 ret = pqos_alloc_assoc_get(cpu_id, &cos_id); 524 #endif 525 if (ret != PQOS_RETVAL_OK) { 526 printf("PQOS: Failed to read COS associated to " 527 "cpu %u on phy_pkg %u.\n", cpu_id, phy_pkg_id); 528 ret = -EFAULT; 529 goto exit; 530 } 531 532 ret = pqos_cpu_get_socketid(m_cpu, cpu_id, &phy_pkg_id); 533 if (ret != PQOS_RETVAL_OK) { 534 printf("PQOS: Failed to get socket for cpu %u\n", 535 cpu_id); 536 ret = -EFAULT; 537 goto exit; 538 } 539 540 /* Mark COS as used */ 541 if (used_cos_table[phy_pkg_id][cos_id] == 0) 542 used_cos_table[phy_pkg_id][cos_id]++; 543 } 544 545 /* look for avail. COS to fulfill requested config */ 546 for (i = 0; i < m_config_count; i++) { 547 for (j = 0; j < m_cpu->num_cores; j++) { 548 cpu_id = m_cpu->cores[j].lcore; 549 if (CPU_ISSET(cpu_id, &m_config[i].cpumask) == 0) 550 continue; 551 552 ret = pqos_cpu_get_socketid(m_cpu, cpu_id, &phy_pkg_id); 553 if (ret != PQOS_RETVAL_OK) { 554 printf("PQOS: Failed to get socket for " 555 "cpu %u\n", cpu_id); 556 ret = -EFAULT; 557 goto exit; 558 } 559 560 /* 561 * Check if we already have COS selected 562 * to be used for that group on that socket 563 */ 564 if (phy_pkg_lcores[phy_pkg_id][i] != 0) 565 continue; 566 567 phy_pkg_lcores[phy_pkg_id][i]++; 568 569 /* Search for avail. COS to be used on that socket */ 570 for (cos_id = 0; cos_id < cos_num; cos_id++) { 571 if (used_cos_table[phy_pkg_id][cos_id] == 0) { 572 used_cos_table[phy_pkg_id][cos_id]++; 573 cos_id_map[i][phy_pkg_id] = cos_id; 574 break; 575 } 576 } 577 578 /* If there is no COS available ...*/ 579 if (cos_id == cos_num) { 580 ret = -E2BIG; 581 goto exit; 582 } 583 } 584 } 585 586 exit: 587 if (ret != 0) 588 printf("PQOS: Not enough available COS to configure " 589 "requested configuration.\n"); 590 591 return ret; 592 } 593 594 static int 595 configure_cat(unsigned cos_id_map[][PQOS_MAX_SOCKETS]) 596 { 597 unsigned phy_pkg_id = 0; 598 unsigned cpu_id = 0; 599 unsigned cos_id = 0; 600 unsigned i = 0; 601 unsigned j = 0; 602 struct pqos_l3ca l3ca = {0}; 603 int ret = 0; 604 605 for (i = 0; i < m_config_count; i++) { 606 memset(&l3ca, 0, sizeof(l3ca)); 607 608 l3ca.cdp = m_config[i].cdp; 609 if (m_config[i].cdp == 1) { 610 #if PQOS_VERSION <= 103 611 l3ca.code_mask = m_config[i].code_mask; 612 l3ca.data_mask = m_config[i].data_mask; 613 #else 614 l3ca.u.s.code_mask = m_config[i].code_mask; 615 l3ca.u.s.data_mask = m_config[i].data_mask; 616 #endif 617 } else 618 #if PQOS_VERSION <= 103 619 l3ca.ways_mask = m_config[i].mask; 620 #else 621 l3ca.u.ways_mask = m_config[i].mask; 622 #endif 623 624 for (j = 0; j < m_sock_count; j++) { 625 phy_pkg_id = m_sockets[j]; 626 if (cos_id_map[i][phy_pkg_id] == 0) 627 continue; 628 629 l3ca.class_id = cos_id_map[i][phy_pkg_id]; 630 631 ret = pqos_l3ca_set(phy_pkg_id, 1, &l3ca); 632 if (ret != PQOS_RETVAL_OK) { 633 printf("PQOS: Failed to set COS %u on " 634 "phy_pkg %u.\n", l3ca.class_id, 635 phy_pkg_id); 636 ret = -EFAULT; 637 goto exit; 638 } 639 } 640 } 641 642 for (i = 0; i < m_config_count; i++) { 643 for (j = 0; j < m_cpu->num_cores; j++) { 644 cpu_id = m_cpu->cores[j].lcore; 645 if (CPU_ISSET(cpu_id, &m_config[i].cpumask) == 0) 646 continue; 647 648 ret = pqos_cpu_get_socketid(m_cpu, cpu_id, &phy_pkg_id); 649 if (ret != PQOS_RETVAL_OK) { 650 printf("PQOS: Failed to get socket for " 651 "cpu %u\n", cpu_id); 652 ret = -EFAULT; 653 goto exit; 654 } 655 656 cos_id = cos_id_map[i][phy_pkg_id]; 657 658 #if PQOS_VERSION <= 103 659 ret = pqos_l3ca_assoc_set(cpu_id, cos_id); 660 #else 661 ret = pqos_alloc_assoc_set(cpu_id, cos_id); 662 #endif 663 if (ret != PQOS_RETVAL_OK) { 664 printf("PQOS: Failed to associate COS %u to " 665 "cpu %u\n", cos_id, cpu_id); 666 ret = -EFAULT; 667 goto exit; 668 } 669 } 670 } 671 672 exit: 673 return ret; 674 } 675 676 677 /* Parse the argument given in the command line of the application */ 678 static int 679 parse_args(int argc, char **argv) 680 { 681 int opt = 0; 682 int retval = 0; 683 int oldopterr = 0; 684 char **argvopt = argv; 685 char *prgname = argv[0]; 686 687 static struct option lgopts[] = { 688 { "l3ca", required_argument, 0, 0 }, 689 { NULL, 0, 0, 0 } 690 }; 691 692 /* Disable printing messages within getopt() */ 693 oldopterr = opterr; 694 opterr = 0; 695 696 opt = getopt_long(argc, argvopt, "", lgopts, NULL); 697 if (opt == 0) { 698 retval = parse_l3ca(optarg); 699 if (retval != 0) { 700 printf("PQOS: Invalid L3CA parameters!\n"); 701 goto exit; 702 } 703 704 argv[optind - 1] = prgname; 705 retval = optind - 1; 706 } else 707 retval = 0; 708 709 exit: 710 /* reset getopt lib */ 711 optind = 1; 712 713 /* Restore opterr value */ 714 opterr = oldopterr; 715 716 return retval; 717 } 718 719 static void 720 print_cmd_line_config(void) 721 { 722 char cpustr[PQOS_MAX_CORES * 3] = {0}; 723 unsigned i = 0; 724 unsigned j = 0; 725 726 for (i = 0; i < m_config_count; i++) { 727 unsigned len = 0; 728 memset(cpustr, 0, sizeof(cpustr)); 729 730 /* Generate CPU list */ 731 for (j = 0; j < PQOS_MAX_CORES; j++) { 732 if (CPU_ISSET(j, &m_config[i].cpumask) != 1) 733 continue; 734 735 len += snprintf(cpustr + len, sizeof(cpustr) - len - 1, 736 "%u,", j); 737 738 if (len >= sizeof(cpustr) - 1) 739 break; 740 } 741 742 if (m_config[i].cdp == 1) { 743 printf("PQOS: CPUs: %s cMASK: 0x%llx, dMASK: " 744 "0x%llx\n", cpustr, 745 (unsigned long long)m_config[i].code_mask, 746 (unsigned long long)m_config[i].data_mask); 747 } else { 748 printf("PQOS: CPUs: %s MASK: 0x%llx\n", cpustr, 749 (unsigned long long)m_config[i].mask); 750 } 751 } 752 } 753 754 /** 755 * @brief Prints CAT configuration 756 */ 757 static void 758 print_cat_config(void) 759 { 760 int ret = PQOS_RETVAL_OK; 761 unsigned i = 0; 762 763 for (i = 0; i < m_sock_count; i++) { 764 struct pqos_l3ca tab[PQOS_MAX_L3CA_COS] = {{0} }; 765 unsigned num = 0; 766 unsigned n = 0; 767 768 ret = pqos_l3ca_get(m_sockets[i], PQOS_MAX_L3CA_COS, &num, tab); 769 if (ret != PQOS_RETVAL_OK) { 770 printf("PQOS: Error retrieving COS!\n"); 771 return; 772 } 773 774 printf("PQOS: COS definitions for Socket %u:\n", m_sockets[i]); 775 for (n = 0; n < num; n++) { 776 if (tab[n].cdp == 1) { 777 printf("PQOS: COS: %u, cMASK: 0x%llx, " 778 "dMASK: 0x%llx\n", tab[n].class_id, 779 #if PQOS_VERSION <= 103 780 (unsigned long long)tab[n].code_mask, 781 (unsigned long long)tab[n].data_mask); 782 #else 783 (unsigned long long)tab[n].u.s.code_mask, 784 (unsigned long long)tab[n].u.s.data_mask); 785 #endif 786 } else { 787 printf("PQOS: COS: %u, MASK: 0x%llx\n", 788 tab[n].class_id, 789 #if PQOS_VERSION <= 103 790 (unsigned long long)tab[n].ways_mask); 791 #else 792 (unsigned long long)tab[n].u.ways_mask); 793 #endif 794 } 795 } 796 } 797 798 for (i = 0; i < m_sock_count; i++) { 799 #if PQOS_VERSION <= 103 800 unsigned lcores[PQOS_MAX_SOCKET_CORES] = {0}; 801 #else 802 unsigned int *lcores = NULL; 803 #endif 804 unsigned lcount = 0; 805 unsigned n = 0; 806 807 #if PQOS_VERSION <= 103 808 ret = pqos_cpu_get_cores(m_cpu, m_sockets[i], 809 PQOS_MAX_SOCKET_CORES, &lcount, &lcores[0]); 810 if (ret != PQOS_RETVAL_OK) { 811 #else 812 lcores = pqos_cpu_get_cores(m_cpu, m_sockets[i], 813 &lcount); 814 if (lcores == NULL || lcount == 0) { 815 #endif 816 printf("PQOS: Error retrieving core information!\n"); 817 return; 818 } 819 820 printf("PQOS: CPU information for socket %u:\n", m_sockets[i]); 821 for (n = 0; n < lcount; n++) { 822 unsigned class_id = 0; 823 824 #if PQOS_VERSION <= 103 825 ret = pqos_l3ca_assoc_get(lcores[n], &class_id); 826 #else 827 ret = pqos_alloc_assoc_get(lcores[n], &class_id); 828 #endif 829 if (ret == PQOS_RETVAL_OK) 830 printf("PQOS: CPU: %u, COS: %u\n", lcores[n], 831 class_id); 832 else 833 printf("PQOS: CPU: %u, ERROR\n", lcores[n]); 834 } 835 836 #if PQOS_VERSION > 103 837 free(lcores); 838 #endif 839 } 840 841 } 842 843 static int 844 cat_validate(void) 845 { 846 int ret = 0; 847 848 ret = check_cpus(); 849 if (ret != 0) 850 return ret; 851 852 ret = check_cdp(); 853 if (ret != 0) 854 return ret; 855 856 ret = check_cbm_len_and_contention(); 857 if (ret != 0) 858 return ret; 859 860 ret = check_cpus_overlapping(); 861 if (ret != 0) 862 return ret; 863 864 return 0; 865 } 866 867 static int 868 cat_set(void) 869 { 870 int ret = 0; 871 unsigned cos_id_map[m_config_count][PQOS_MAX_SOCKETS]; 872 873 memset(cos_id_map, 0, sizeof(cos_id_map)); 874 875 ret = check_and_select_classes(cos_id_map); 876 if (ret != 0) 877 return ret; 878 879 ret = configure_cat(cos_id_map); 880 if (ret != 0) 881 return ret; 882 883 return 0; 884 } 885 886 static void 887 cat_fini(void) 888 { 889 int ret = 0; 890 891 printf("PQOS: Shutting down PQoS library...\n"); 892 893 /* deallocate all the resources */ 894 ret = pqos_fini(); 895 if (ret != PQOS_RETVAL_OK && ret != PQOS_RETVAL_INIT) 896 printf("PQOS: Error shutting down PQoS library!\n"); 897 898 m_cap = NULL; 899 m_cpu = NULL; 900 m_cap_l3ca = NULL; 901 #if PQOS_VERSION <= 103 902 memset(m_sockets, 0, sizeof(m_sockets)); 903 #else 904 if (m_sockets != NULL) 905 free(m_sockets); 906 #endif 907 m_sock_count = 0; 908 memset(m_config, 0, sizeof(m_config)); 909 m_config_count = 0; 910 } 911 912 void 913 cat_exit(void) 914 { 915 unsigned i = 0; 916 unsigned j = 0; 917 unsigned cpu_id = 0; 918 int ret = 0; 919 920 /* if lib is not initialized, do nothing */ 921 if (m_cap == NULL && m_cpu == NULL) 922 return; 923 924 printf("PQOS: Reverting CAT configuration...\n"); 925 926 for (i = 0; i < m_config_count; i++) { 927 for (j = 0; j < m_cpu->num_cores; j++) { 928 cpu_id = m_cpu->cores[j].lcore; 929 if (CPU_ISSET(cpu_id, &m_config[i].cpumask) == 0) 930 continue; 931 932 #if PQOS_VERSION <= 103 933 ret = pqos_l3ca_assoc_set(cpu_id, 0); 934 #else 935 ret = pqos_alloc_assoc_set(cpu_id, 0); 936 #endif 937 if (ret != PQOS_RETVAL_OK) { 938 printf("PQOS: Failed to associate COS 0 to " 939 "cpu %u\n", cpu_id); 940 } 941 } 942 } 943 944 cat_fini(); 945 } 946 947 static void 948 signal_handler(int signum) 949 { 950 if (signum == SIGINT || signum == SIGTERM) { 951 printf("\nPQOS: Signal %d received, preparing to exit...\n", 952 signum); 953 954 cat_exit(); 955 956 /* exit with the expected status */ 957 signal(signum, SIG_DFL); 958 kill(getpid(), signum); 959 } 960 } 961 962 int 963 cat_init(int argc, char **argv) 964 { 965 int ret = 0; 966 int args_num = 0; 967 struct pqos_config cfg = {0}; 968 969 if (m_cap != NULL || m_cpu != NULL) { 970 printf("PQOS: CAT module already initialized!\n"); 971 return -EEXIST; 972 } 973 974 /* Parse cmd line args */ 975 ret = parse_args(argc, argv); 976 977 if (ret <= 0) 978 goto err; 979 980 args_num = ret; 981 982 /* Print cmd line configuration */ 983 print_cmd_line_config(); 984 985 /* PQoS Initialization - Check and initialize CAT capability */ 986 cfg.fd_log = STDOUT_FILENO; 987 cfg.verbose = 0; 988 #if PQOS_VERSION <= 103 989 cfg.cdp_cfg = PQOS_REQUIRE_CDP_ANY; 990 #endif 991 ret = pqos_init(&cfg); 992 if (ret != PQOS_RETVAL_OK) { 993 printf("PQOS: Error initializing PQoS library!\n"); 994 ret = -EFAULT; 995 goto err; 996 } 997 998 /* Get capability and CPU info pointer */ 999 ret = pqos_cap_get(&m_cap, &m_cpu); 1000 if (ret != PQOS_RETVAL_OK || m_cap == NULL || m_cpu == NULL) { 1001 printf("PQOS: Error retrieving PQoS capabilities!\n"); 1002 ret = -EFAULT; 1003 goto err; 1004 } 1005 1006 /* Get L3CA capabilities */ 1007 ret = pqos_cap_get_type(m_cap, PQOS_CAP_TYPE_L3CA, &m_cap_l3ca); 1008 if (ret != PQOS_RETVAL_OK || m_cap_l3ca == NULL) { 1009 printf("PQOS: Error retrieving PQOS_CAP_TYPE_L3CA " 1010 "capabilities!\n"); 1011 ret = -EFAULT; 1012 goto err; 1013 } 1014 1015 /* Get CPU socket information */ 1016 #if PQOS_VERSION <= 103 1017 ret = pqos_cpu_get_sockets(m_cpu, PQOS_MAX_SOCKETS, &m_sock_count, 1018 m_sockets); 1019 if (ret != PQOS_RETVAL_OK) { 1020 #else 1021 m_sockets = pqos_cpu_get_sockets(m_cpu, &m_sock_count); 1022 if (m_sockets == NULL) { 1023 #endif 1024 printf("PQOS: Error retrieving CPU socket information!\n"); 1025 ret = -EFAULT; 1026 goto err; 1027 } 1028 1029 /* Validate cmd line configuration */ 1030 ret = cat_validate(); 1031 if (ret != 0) { 1032 printf("PQOS: Requested CAT configuration is not valid!\n"); 1033 goto err; 1034 } 1035 1036 /* configure system */ 1037 ret = cat_set(); 1038 if (ret != 0) { 1039 printf("PQOS: Failed to configure CAT!\n"); 1040 goto err; 1041 } 1042 1043 signal(SIGINT, signal_handler); 1044 signal(SIGTERM, signal_handler); 1045 1046 ret = atexit(cat_exit); 1047 if (ret != 0) { 1048 printf("PQOS: Cannot set exit function\n"); 1049 goto err; 1050 } 1051 1052 /* Print CAT configuration */ 1053 print_cat_config(); 1054 1055 return args_num; 1056 1057 err: 1058 /* deallocate all the resources */ 1059 cat_fini(); 1060 return ret; 1061 } 1062