1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Netronome Systems, Inc. 3 * All rights reserved. 4 */ 5 6 #ifndef __NFP_CPPAT_H__ 7 #define __NFP_CPPAT_H__ 8 9 #include "nfp_platform.h" 10 #include "nfp_resid.h" 11 12 /* This file contains helpers for creating CPP commands 13 * 14 * All magic NFP-6xxx IMB 'mode' numbers here are from: 15 * Databook (1 August 2013) 16 * - System Overview and Connectivity 17 * -- Internal Connectivity 18 * --- Distributed Switch Fabric - Command Push/Pull (DSF-CPP) Bus 19 * ---- CPP addressing 20 * ----- Table 3.6. CPP Address Translation Mode Commands 21 */ 22 23 #define _NIC_NFP6000_MU_LOCALITY_DIRECT 2 24 25 static inline int 26 _nfp6000_decode_basic(uint64_t addr, int *dest_island, int cpp_tgt, int mode, 27 int addr40, int isld1, int isld0); 28 29 static uint64_t 30 _nic_mask64(int msb, int lsb, int at0) 31 { 32 uint64_t v; 33 int w = msb - lsb + 1; 34 35 if (w == 64) 36 return ~(uint64_t)0; 37 38 if ((lsb + w) > 64) 39 return 0; 40 41 v = (UINT64_C(1) << w) - 1; 42 43 if (at0) 44 return v; 45 46 return v << lsb; 47 } 48 49 /* For VQDR, we may not modify the Channel bits, which might overlap 50 * with the Index bit. When it does, we need to ensure that isld0 == isld1. 51 */ 52 static inline int 53 _nfp6000_encode_basic(uint64_t *addr, int dest_island, int cpp_tgt, int mode, 54 int addr40, int isld1, int isld0) 55 { 56 uint64_t _u64; 57 int iid_lsb, idx_lsb; 58 int i, v = 0; 59 int isld[2]; 60 61 isld[0] = isld0; 62 isld[1] = isld1; 63 64 switch (cpp_tgt) { 65 case NFP6000_CPPTGT_MU: 66 /* This function doesn't handle MU */ 67 return NFP_ERRNO(EINVAL); 68 case NFP6000_CPPTGT_CTXPB: 69 /* This function doesn't handle CTXPB */ 70 return NFP_ERRNO(EINVAL); 71 default: 72 break; 73 } 74 75 switch (mode) { 76 case 0: 77 if (cpp_tgt == NFP6000_CPPTGT_VQDR && !addr40) { 78 /* 79 * In this specific mode we'd rather not modify the 80 * address but we can verify if the existing contents 81 * will point to a valid island. 82 */ 83 i = _nfp6000_decode_basic(*addr, &v, cpp_tgt, mode, 84 addr40, isld1, 85 isld0); 86 if (i != 0) 87 /* Full Island ID and channel bits overlap */ 88 return i; 89 90 /* 91 * If dest_island is invalid, the current address won't 92 * go where expected. 93 */ 94 if (dest_island != -1 && dest_island != v) 95 return NFP_ERRNO(EINVAL); 96 97 /* If dest_island was -1, we don't care */ 98 return 0; 99 } 100 101 iid_lsb = (addr40) ? 34 : 26; 102 103 /* <39:34> or <31:26> */ 104 _u64 = _nic_mask64((iid_lsb + 5), iid_lsb, 0); 105 *addr &= ~_u64; 106 *addr |= (((uint64_t)dest_island) << iid_lsb) & _u64; 107 return 0; 108 case 1: 109 if (cpp_tgt == NFP6000_CPPTGT_VQDR && !addr40) { 110 i = _nfp6000_decode_basic(*addr, &v, cpp_tgt, mode, 111 addr40, isld1, isld0); 112 if (i != 0) 113 /* Full Island ID and channel bits overlap */ 114 return i; 115 116 /* 117 * If dest_island is invalid, the current address won't 118 * go where expected. 119 */ 120 if (dest_island != -1 && dest_island != v) 121 return NFP_ERRNO(EINVAL); 122 123 /* If dest_island was -1, we don't care */ 124 return 0; 125 } 126 127 idx_lsb = (addr40) ? 39 : 31; 128 if (dest_island == isld0) { 129 /* Only need to clear the Index bit */ 130 *addr &= ~_nic_mask64(idx_lsb, idx_lsb, 0); 131 return 0; 132 } 133 134 if (dest_island == isld1) { 135 /* Only need to set the Index bit */ 136 *addr |= (UINT64_C(1) << idx_lsb); 137 return 0; 138 } 139 140 return NFP_ERRNO(ENODEV); 141 case 2: 142 if (cpp_tgt == NFP6000_CPPTGT_VQDR && !addr40) { 143 /* iid<0> = addr<30> = channel<0> */ 144 /* channel<1> = addr<31> = Index */ 145 146 /* 147 * Special case where we allow channel bits to be set 148 * before hand and with them select an island. 149 * So we need to confirm that it's at least plausible. 150 */ 151 i = _nfp6000_decode_basic(*addr, &v, cpp_tgt, mode, 152 addr40, isld1, isld0); 153 if (i != 0) 154 /* Full Island ID and channel bits overlap */ 155 return i; 156 157 /* 158 * If dest_island is invalid, the current address won't 159 * go where expected. 160 */ 161 if (dest_island != -1 && dest_island != v) 162 return NFP_ERRNO(EINVAL); 163 164 /* If dest_island was -1, we don't care */ 165 return 0; 166 } 167 168 /* 169 * Make sure we compare against isldN values by clearing the 170 * LSB. This is what the silicon does. 171 **/ 172 isld[0] &= ~1; 173 isld[1] &= ~1; 174 175 idx_lsb = (addr40) ? 39 : 31; 176 iid_lsb = idx_lsb - 1; 177 178 /* 179 * Try each option, take first one that fits. Not sure if we 180 * would want to do some smarter searching and prefer 0 or non-0 181 * island IDs. 182 */ 183 184 for (i = 0; i < 2; i++) { 185 for (v = 0; v < 2; v++) { 186 if (dest_island != (isld[i] | v)) 187 continue; 188 *addr &= ~_nic_mask64(idx_lsb, iid_lsb, 0); 189 *addr |= (((uint64_t)i) << idx_lsb); 190 *addr |= (((uint64_t)v) << iid_lsb); 191 return 0; 192 } 193 } 194 195 return NFP_ERRNO(ENODEV); 196 case 3: 197 if (cpp_tgt == NFP6000_CPPTGT_VQDR && !addr40) { 198 /* 199 * iid<0> = addr<29> = data 200 * iid<1> = addr<30> = channel<0> 201 * channel<1> = addr<31> = Index 202 */ 203 i = _nfp6000_decode_basic(*addr, &v, cpp_tgt, mode, 204 addr40, isld1, isld0); 205 if (i != 0) 206 /* Full Island ID and channel bits overlap */ 207 return i; 208 209 if (dest_island != -1 && dest_island != v) 210 return NFP_ERRNO(EINVAL); 211 212 /* If dest_island was -1, we don't care */ 213 return 0; 214 } 215 216 isld[0] &= ~3; 217 isld[1] &= ~3; 218 219 idx_lsb = (addr40) ? 39 : 31; 220 iid_lsb = idx_lsb - 2; 221 222 for (i = 0; i < 2; i++) { 223 for (v = 0; v < 4; v++) { 224 if (dest_island != (isld[i] | v)) 225 continue; 226 *addr &= ~_nic_mask64(idx_lsb, iid_lsb, 0); 227 *addr |= (((uint64_t)i) << idx_lsb); 228 *addr |= (((uint64_t)v) << iid_lsb); 229 return 0; 230 } 231 } 232 return NFP_ERRNO(ENODEV); 233 default: 234 break; 235 } 236 237 return NFP_ERRNO(EINVAL); 238 } 239 240 static inline int 241 _nfp6000_decode_basic(uint64_t addr, int *dest_island, int cpp_tgt, int mode, 242 int addr40, int isld1, int isld0) 243 { 244 int iid_lsb, idx_lsb; 245 246 switch (cpp_tgt) { 247 case NFP6000_CPPTGT_MU: 248 /* This function doesn't handle MU */ 249 return NFP_ERRNO(EINVAL); 250 case NFP6000_CPPTGT_CTXPB: 251 /* This function doesn't handle CTXPB */ 252 return NFP_ERRNO(EINVAL); 253 default: 254 break; 255 } 256 257 switch (mode) { 258 case 0: 259 /* 260 * For VQDR, in this mode for 32-bit addressing it would be 261 * islands 0, 16, 32 and 48 depending on channel and upper 262 * address bits. Since those are not all valid islands, most 263 * decode cases would result in bad island IDs, but we do them 264 * anyway since this is decoding an address that is already 265 * assumed to be used as-is to get to sram. 266 */ 267 iid_lsb = (addr40) ? 34 : 26; 268 *dest_island = (int)(addr >> iid_lsb) & 0x3F; 269 return 0; 270 case 1: 271 /* 272 * For VQDR 32-bit, this would decode as: 273 * Channel 0: island#0 274 * Channel 1: island#0 275 * Channel 2: island#1 276 * Channel 3: island#1 277 * 278 * That would be valid as long as both islands have VQDR. 279 * Let's allow this. 280 */ 281 282 idx_lsb = (addr40) ? 39 : 31; 283 if (addr & _nic_mask64(idx_lsb, idx_lsb, 0)) 284 *dest_island = isld1; 285 else 286 *dest_island = isld0; 287 288 return 0; 289 case 2: 290 /* 291 * For VQDR 32-bit: 292 * Channel 0: (island#0 | 0) 293 * Channel 1: (island#0 | 1) 294 * Channel 2: (island#1 | 0) 295 * Channel 3: (island#1 | 1) 296 * 297 * Make sure we compare against isldN values by clearing the 298 * LSB. This is what the silicon does. 299 */ 300 isld0 &= ~1; 301 isld1 &= ~1; 302 303 idx_lsb = (addr40) ? 39 : 31; 304 iid_lsb = idx_lsb - 1; 305 306 if (addr & _nic_mask64(idx_lsb, idx_lsb, 0)) 307 *dest_island = isld1 | (int)((addr >> iid_lsb) & 1); 308 else 309 *dest_island = isld0 | (int)((addr >> iid_lsb) & 1); 310 311 return 0; 312 case 3: 313 /* 314 * In this mode the data address starts to affect the island ID 315 * so rather not allow it. In some really specific case one 316 * could use this to send the upper half of the VQDR channel to 317 * another MU, but this is getting very specific. However, as 318 * above for mode 0, this is the decoder and the caller should 319 * validate the resulting IID. This blindly does what the 320 * silicon would do. 321 */ 322 323 isld0 &= ~3; 324 isld1 &= ~3; 325 326 idx_lsb = (addr40) ? 39 : 31; 327 iid_lsb = idx_lsb - 2; 328 329 if (addr & _nic_mask64(idx_lsb, idx_lsb, 0)) 330 *dest_island = isld1 | (int)((addr >> iid_lsb) & 3); 331 else 332 *dest_island = isld0 | (int)((addr >> iid_lsb) & 3); 333 334 return 0; 335 default: 336 break; 337 } 338 339 return NFP_ERRNO(EINVAL); 340 } 341 342 static inline int 343 _nfp6000_cppat_mu_locality_lsb(int mode, int addr40) 344 { 345 switch (mode) { 346 case 0: 347 case 1: 348 case 2: 349 case 3: 350 return (addr40) ? 38 : 30; 351 default: 352 break; 353 } 354 return NFP_ERRNO(EINVAL); 355 } 356 357 static inline int 358 _nfp6000_encode_mu(uint64_t *addr, int dest_island, int mode, int addr40, 359 int isld1, int isld0) 360 { 361 uint64_t _u64; 362 int iid_lsb, idx_lsb, locality_lsb; 363 int i, v; 364 int isld[2]; 365 int da; 366 367 isld[0] = isld0; 368 isld[1] = isld1; 369 locality_lsb = _nfp6000_cppat_mu_locality_lsb(mode, addr40); 370 371 if (locality_lsb < 0) 372 return NFP_ERRNO(EINVAL); 373 374 if (((*addr >> locality_lsb) & 3) == _NIC_NFP6000_MU_LOCALITY_DIRECT) 375 da = 1; 376 else 377 da = 0; 378 379 switch (mode) { 380 case 0: 381 iid_lsb = (addr40) ? 32 : 24; 382 _u64 = _nic_mask64((iid_lsb + 5), iid_lsb, 0); 383 *addr &= ~_u64; 384 *addr |= (((uint64_t)dest_island) << iid_lsb) & _u64; 385 return 0; 386 case 1: 387 if (da) { 388 iid_lsb = (addr40) ? 32 : 24; 389 _u64 = _nic_mask64((iid_lsb + 5), iid_lsb, 0); 390 *addr &= ~_u64; 391 *addr |= (((uint64_t)dest_island) << iid_lsb) & _u64; 392 return 0; 393 } 394 395 idx_lsb = (addr40) ? 37 : 29; 396 if (dest_island == isld0) { 397 *addr &= ~_nic_mask64(idx_lsb, idx_lsb, 0); 398 return 0; 399 } 400 401 if (dest_island == isld1) { 402 *addr |= (UINT64_C(1) << idx_lsb); 403 return 0; 404 } 405 406 return NFP_ERRNO(ENODEV); 407 case 2: 408 if (da) { 409 iid_lsb = (addr40) ? 32 : 24; 410 _u64 = _nic_mask64((iid_lsb + 5), iid_lsb, 0); 411 *addr &= ~_u64; 412 *addr |= (((uint64_t)dest_island) << iid_lsb) & _u64; 413 return 0; 414 } 415 416 /* 417 * Make sure we compare against isldN values by clearing the 418 * LSB. This is what the silicon does. 419 */ 420 isld[0] &= ~1; 421 isld[1] &= ~1; 422 423 idx_lsb = (addr40) ? 37 : 29; 424 iid_lsb = idx_lsb - 1; 425 426 /* 427 * Try each option, take first one that fits. Not sure if we 428 * would want to do some smarter searching and prefer 0 or 429 * non-0 island IDs. 430 */ 431 432 for (i = 0; i < 2; i++) { 433 for (v = 0; v < 2; v++) { 434 if (dest_island != (isld[i] | v)) 435 continue; 436 *addr &= ~_nic_mask64(idx_lsb, iid_lsb, 0); 437 *addr |= (((uint64_t)i) << idx_lsb); 438 *addr |= (((uint64_t)v) << iid_lsb); 439 return 0; 440 } 441 } 442 return NFP_ERRNO(ENODEV); 443 case 3: 444 /* 445 * Only the EMU will use 40 bit addressing. Silently set the 446 * direct locality bit for everyone else. The SDK toolchain 447 * uses dest_island <= 0 to test for atypical address encodings 448 * to support access to local-island CTM with a 32-but address 449 * (high-locality is effectively ignored and just used for 450 * routing to island #0). 451 */ 452 if (dest_island > 0 && 453 (dest_island < 24 || dest_island > 26)) { 454 *addr |= ((uint64_t)_NIC_NFP6000_MU_LOCALITY_DIRECT) 455 << locality_lsb; 456 da = 1; 457 } 458 459 if (da) { 460 iid_lsb = (addr40) ? 32 : 24; 461 _u64 = _nic_mask64((iid_lsb + 5), iid_lsb, 0); 462 *addr &= ~_u64; 463 *addr |= (((uint64_t)dest_island) << iid_lsb) & _u64; 464 return 0; 465 } 466 467 isld[0] &= ~3; 468 isld[1] &= ~3; 469 470 idx_lsb = (addr40) ? 37 : 29; 471 iid_lsb = idx_lsb - 2; 472 473 for (i = 0; i < 2; i++) { 474 for (v = 0; v < 4; v++) { 475 if (dest_island != (isld[i] | v)) 476 continue; 477 *addr &= ~_nic_mask64(idx_lsb, iid_lsb, 0); 478 *addr |= (((uint64_t)i) << idx_lsb); 479 *addr |= (((uint64_t)v) << iid_lsb); 480 return 0; 481 } 482 } 483 484 return NFP_ERRNO(ENODEV); 485 default: 486 break; 487 } 488 489 return NFP_ERRNO(EINVAL); 490 } 491 492 static inline int 493 _nfp6000_decode_mu(uint64_t addr, int *dest_island, int mode, int addr40, 494 int isld1, int isld0) 495 { 496 int iid_lsb, idx_lsb, locality_lsb; 497 int da; 498 499 locality_lsb = _nfp6000_cppat_mu_locality_lsb(mode, addr40); 500 501 if (((addr >> locality_lsb) & 3) == _NIC_NFP6000_MU_LOCALITY_DIRECT) 502 da = 1; 503 else 504 da = 0; 505 506 switch (mode) { 507 case 0: 508 iid_lsb = (addr40) ? 32 : 24; 509 *dest_island = (int)(addr >> iid_lsb) & 0x3F; 510 return 0; 511 case 1: 512 if (da) { 513 iid_lsb = (addr40) ? 32 : 24; 514 *dest_island = (int)(addr >> iid_lsb) & 0x3F; 515 return 0; 516 } 517 518 idx_lsb = (addr40) ? 37 : 29; 519 520 if (addr & _nic_mask64(idx_lsb, idx_lsb, 0)) 521 *dest_island = isld1; 522 else 523 *dest_island = isld0; 524 525 return 0; 526 case 2: 527 if (da) { 528 iid_lsb = (addr40) ? 32 : 24; 529 *dest_island = (int)(addr >> iid_lsb) & 0x3F; 530 return 0; 531 } 532 /* 533 * Make sure we compare against isldN values by clearing the 534 * LSB. This is what the silicon does. 535 */ 536 isld0 &= ~1; 537 isld1 &= ~1; 538 539 idx_lsb = (addr40) ? 37 : 29; 540 iid_lsb = idx_lsb - 1; 541 542 if (addr & _nic_mask64(idx_lsb, idx_lsb, 0)) 543 *dest_island = isld1 | (int)((addr >> iid_lsb) & 1); 544 else 545 *dest_island = isld0 | (int)((addr >> iid_lsb) & 1); 546 547 return 0; 548 case 3: 549 if (da) { 550 iid_lsb = (addr40) ? 32 : 24; 551 *dest_island = (int)(addr >> iid_lsb) & 0x3F; 552 return 0; 553 } 554 555 isld0 &= ~3; 556 isld1 &= ~3; 557 558 idx_lsb = (addr40) ? 37 : 29; 559 iid_lsb = idx_lsb - 2; 560 561 if (addr & _nic_mask64(idx_lsb, idx_lsb, 0)) 562 *dest_island = isld1 | (int)((addr >> iid_lsb) & 3); 563 else 564 *dest_island = isld0 | (int)((addr >> iid_lsb) & 3); 565 566 return 0; 567 default: 568 break; 569 } 570 571 return NFP_ERRNO(EINVAL); 572 } 573 574 static inline int 575 _nfp6000_cppat_addr_encode(uint64_t *addr, int dest_island, int cpp_tgt, 576 int mode, int addr40, int isld1, int isld0) 577 { 578 switch (cpp_tgt) { 579 case NFP6000_CPPTGT_NBI: 580 case NFP6000_CPPTGT_VQDR: 581 case NFP6000_CPPTGT_ILA: 582 case NFP6000_CPPTGT_PCIE: 583 case NFP6000_CPPTGT_ARM: 584 case NFP6000_CPPTGT_CRYPTO: 585 case NFP6000_CPPTGT_CLS: 586 return _nfp6000_encode_basic(addr, dest_island, cpp_tgt, mode, 587 addr40, isld1, isld0); 588 589 case NFP6000_CPPTGT_MU: 590 return _nfp6000_encode_mu(addr, dest_island, mode, addr40, 591 isld1, isld0); 592 593 case NFP6000_CPPTGT_CTXPB: 594 if (mode != 1 || addr40 != 0) 595 return NFP_ERRNO(EINVAL); 596 597 *addr &= ~_nic_mask64(29, 24, 0); 598 *addr |= (((uint64_t)dest_island) << 24) & 599 _nic_mask64(29, 24, 0); 600 return 0; 601 default: 602 break; 603 } 604 605 return NFP_ERRNO(EINVAL); 606 } 607 608 static inline int 609 _nfp6000_cppat_addr_decode(uint64_t addr, int *dest_island, int cpp_tgt, 610 int mode, int addr40, int isld1, int isld0) 611 { 612 switch (cpp_tgt) { 613 case NFP6000_CPPTGT_NBI: 614 case NFP6000_CPPTGT_VQDR: 615 case NFP6000_CPPTGT_ILA: 616 case NFP6000_CPPTGT_PCIE: 617 case NFP6000_CPPTGT_ARM: 618 case NFP6000_CPPTGT_CRYPTO: 619 case NFP6000_CPPTGT_CLS: 620 return _nfp6000_decode_basic(addr, dest_island, cpp_tgt, mode, 621 addr40, isld1, isld0); 622 623 case NFP6000_CPPTGT_MU: 624 return _nfp6000_decode_mu(addr, dest_island, mode, addr40, 625 isld1, isld0); 626 627 case NFP6000_CPPTGT_CTXPB: 628 if (mode != 1 || addr40 != 0) 629 return -EINVAL; 630 *dest_island = (int)(addr >> 24) & 0x3F; 631 return 0; 632 default: 633 break; 634 } 635 636 return -EINVAL; 637 } 638 639 static inline int 640 _nfp6000_cppat_addr_iid_clear(uint64_t *addr, int cpp_tgt, int mode, int addr40) 641 { 642 int iid_lsb, locality_lsb, da; 643 644 switch (cpp_tgt) { 645 case NFP6000_CPPTGT_NBI: 646 case NFP6000_CPPTGT_VQDR: 647 case NFP6000_CPPTGT_ILA: 648 case NFP6000_CPPTGT_PCIE: 649 case NFP6000_CPPTGT_ARM: 650 case NFP6000_CPPTGT_CRYPTO: 651 case NFP6000_CPPTGT_CLS: 652 switch (mode) { 653 case 0: 654 iid_lsb = (addr40) ? 34 : 26; 655 *addr &= ~(UINT64_C(0x3F) << iid_lsb); 656 return 0; 657 case 1: 658 iid_lsb = (addr40) ? 39 : 31; 659 *addr &= ~_nic_mask64(iid_lsb, iid_lsb, 0); 660 return 0; 661 case 2: 662 iid_lsb = (addr40) ? 38 : 30; 663 *addr &= ~_nic_mask64(iid_lsb + 1, iid_lsb, 0); 664 return 0; 665 case 3: 666 iid_lsb = (addr40) ? 37 : 29; 667 *addr &= ~_nic_mask64(iid_lsb + 2, iid_lsb, 0); 668 return 0; 669 default: 670 break; 671 } 672 case NFP6000_CPPTGT_MU: 673 locality_lsb = _nfp6000_cppat_mu_locality_lsb(mode, addr40); 674 da = (((*addr >> locality_lsb) & 3) == 675 _NIC_NFP6000_MU_LOCALITY_DIRECT); 676 switch (mode) { 677 case 0: 678 iid_lsb = (addr40) ? 32 : 24; 679 *addr &= ~(UINT64_C(0x3F) << iid_lsb); 680 return 0; 681 case 1: 682 if (da) { 683 iid_lsb = (addr40) ? 32 : 24; 684 *addr &= ~(UINT64_C(0x3F) << iid_lsb); 685 return 0; 686 } 687 iid_lsb = (addr40) ? 37 : 29; 688 *addr &= ~_nic_mask64(iid_lsb, iid_lsb, 0); 689 return 0; 690 case 2: 691 if (da) { 692 iid_lsb = (addr40) ? 32 : 24; 693 *addr &= ~(UINT64_C(0x3F) << iid_lsb); 694 return 0; 695 } 696 697 iid_lsb = (addr40) ? 36 : 28; 698 *addr &= ~_nic_mask64(iid_lsb + 1, iid_lsb, 0); 699 return 0; 700 case 3: 701 if (da) { 702 iid_lsb = (addr40) ? 32 : 24; 703 *addr &= ~(UINT64_C(0x3F) << iid_lsb); 704 return 0; 705 } 706 707 iid_lsb = (addr40) ? 35 : 27; 708 *addr &= ~_nic_mask64(iid_lsb + 2, iid_lsb, 0); 709 return 0; 710 default: 711 break; 712 } 713 case NFP6000_CPPTGT_CTXPB: 714 if (mode != 1 || addr40 != 0) 715 return 0; 716 *addr &= ~(UINT64_C(0x3F) << 24); 717 return 0; 718 default: 719 break; 720 } 721 722 return NFP_ERRNO(EINVAL); 723 } 724 725 #endif /* __NFP_CPPAT_H__ */ 726