1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 #include "roc_api.h" 5 #include "roc_priv.h" 6 7 static void 8 npc_prep_mcam_ldata(uint8_t *ptr, const uint8_t *data, int len) 9 { 10 int idx; 11 12 for (idx = 0; idx < len; idx++) 13 ptr[idx] = data[len - 1 - idx]; 14 } 15 16 static int 17 npc_check_copysz(size_t size, size_t len) 18 { 19 if (len <= size) 20 return len; 21 return NPC_ERR_PARAM; 22 } 23 24 static inline int 25 npc_mem_is_zero(const void *mem, int len) 26 { 27 const char *m = mem; 28 int i; 29 30 for (i = 0; i < len; i++) { 31 if (m[i] != 0) 32 return 0; 33 } 34 return 1; 35 } 36 37 static void 38 npc_set_hw_mask(struct npc_parse_item_info *info, struct npc_xtract_info *xinfo, 39 char *hw_mask) 40 { 41 int max_off, offset; 42 int j; 43 44 if (xinfo->enable == 0) 45 return; 46 47 if (xinfo->hdr_off < info->hw_hdr_len) 48 return; 49 50 max_off = xinfo->hdr_off + xinfo->len - info->hw_hdr_len; 51 52 if (max_off > info->len) 53 max_off = info->len; 54 55 offset = xinfo->hdr_off - info->hw_hdr_len; 56 for (j = offset; j < max_off; j++) 57 hw_mask[j] = 0xff; 58 } 59 60 void 61 npc_get_hw_supp_mask(struct npc_parse_state *pst, 62 struct npc_parse_item_info *info, int lid, int lt) 63 { 64 struct npc_xtract_info *xinfo, *lfinfo; 65 char *hw_mask = info->hw_mask; 66 int lf_cfg = 0; 67 int i, j; 68 int intf; 69 70 intf = pst->nix_intf; 71 xinfo = pst->npc->prx_dxcfg[intf][lid][lt].xtract; 72 memset(hw_mask, 0, info->len); 73 74 for (i = 0; i < NPC_MAX_LD; i++) 75 npc_set_hw_mask(info, &xinfo[i], hw_mask); 76 77 for (i = 0; i < NPC_MAX_LD; i++) { 78 if (xinfo[i].flags_enable == 0) 79 continue; 80 81 lf_cfg = pst->npc->prx_lfcfg[i].i; 82 if (lf_cfg == lid) { 83 for (j = 0; j < NPC_MAX_LFL; j++) { 84 lfinfo = pst->npc->prx_fxcfg[intf][i][j].xtract; 85 npc_set_hw_mask(info, &lfinfo[0], hw_mask); 86 } 87 } 88 } 89 } 90 91 static inline int 92 npc_mask_is_supported(const char *mask, const char *hw_mask, int len) 93 { 94 /* 95 * If no hw_mask, assume nothing is supported. 96 * mask is never NULL 97 */ 98 if (hw_mask == NULL) 99 return npc_mem_is_zero(mask, len); 100 101 while (len--) { 102 if ((mask[len] | hw_mask[len]) != hw_mask[len]) 103 return 0; /* False */ 104 } 105 return 1; 106 } 107 108 int 109 npc_parse_item_basic(const struct roc_npc_item_info *item, 110 struct npc_parse_item_info *info) 111 { 112 /* Item must not be NULL */ 113 if (item == NULL) 114 return NPC_ERR_PARAM; 115 116 /* Don't support ranges */ 117 if (item->last != NULL) 118 return NPC_ERR_INVALID_RANGE; 119 120 /* If spec is NULL, both mask and last must be NULL, this 121 * makes it to match ANY value (eq to mask = 0). 122 * Setting either mask or last without spec is an error 123 */ 124 if (item->spec == NULL) { 125 if (item->last == NULL && item->mask == NULL) { 126 info->spec = NULL; 127 return 0; 128 } 129 return NPC_ERR_INVALID_SPEC; 130 } 131 132 /* We have valid spec */ 133 info->spec = item->spec; 134 135 /* If mask is not set, use default mask, err if default mask is 136 * also NULL. 137 */ 138 if (item->mask == NULL) { 139 if (info->def_mask == NULL) 140 return NPC_ERR_PARAM; 141 info->mask = info->def_mask; 142 } else { 143 info->mask = item->mask; 144 } 145 146 /* mask specified must be subset of hw supported mask 147 * mask | hw_mask == hw_mask 148 */ 149 if (!npc_mask_is_supported(info->mask, info->hw_mask, info->len)) 150 return NPC_ERR_INVALID_MASK; 151 152 return 0; 153 } 154 155 static int 156 npc_update_extraction_data(struct npc_parse_state *pst, 157 struct npc_parse_item_info *info, 158 struct npc_xtract_info *xinfo) 159 { 160 uint8_t int_info_mask[NPC_MAX_EXTRACT_DATA_LEN]; 161 uint8_t int_info[NPC_MAX_EXTRACT_DATA_LEN]; 162 struct npc_xtract_info *x; 163 int hdr_off; 164 int len = 0; 165 166 x = xinfo; 167 len = x->len; 168 hdr_off = x->hdr_off; 169 170 if (hdr_off < info->hw_hdr_len) 171 return 0; 172 173 if (x->enable == 0) 174 return 0; 175 176 hdr_off -= info->hw_hdr_len; 177 178 if (hdr_off >= info->len) 179 return 0; 180 181 if (hdr_off + len > info->len) 182 len = info->len - hdr_off; 183 184 len = npc_check_copysz((ROC_NPC_MAX_MCAM_WIDTH_DWORDS * 8) - x->key_off, 185 len); 186 if (len < 0) 187 return NPC_ERR_INVALID_SIZE; 188 189 /* Need to reverse complete structure so that dest addr is at 190 * MSB so as to program the MCAM using mcam_data & mcam_mask 191 * arrays 192 */ 193 npc_prep_mcam_ldata(int_info, (const uint8_t *)info->spec + hdr_off, 194 x->len); 195 npc_prep_mcam_ldata(int_info_mask, 196 (const uint8_t *)info->mask + hdr_off, x->len); 197 198 memcpy(pst->mcam_mask + x->key_off, int_info_mask, len); 199 memcpy(pst->mcam_data + x->key_off, int_info, len); 200 return 0; 201 } 202 203 int 204 npc_update_parse_state(struct npc_parse_state *pst, 205 struct npc_parse_item_info *info, int lid, int lt, 206 uint8_t flags) 207 { 208 struct npc_lid_lt_xtract_info *xinfo; 209 struct npc_xtract_info *lfinfo; 210 int intf, lf_cfg; 211 int i, j, rc = 0; 212 213 pst->layer_mask |= lid; 214 pst->lt[lid] = lt; 215 pst->flags[lid] = flags; 216 217 intf = pst->nix_intf; 218 xinfo = &pst->npc->prx_dxcfg[intf][lid][lt]; 219 if (xinfo->is_terminating) 220 pst->terminate = 1; 221 222 if (info->spec == NULL) 223 goto done; 224 225 for (i = 0; i < NPC_MAX_LD; i++) { 226 rc = npc_update_extraction_data(pst, info, &xinfo->xtract[i]); 227 if (rc != 0) 228 return rc; 229 } 230 231 for (i = 0; i < NPC_MAX_LD; i++) { 232 if (xinfo->xtract[i].flags_enable == 0) 233 continue; 234 235 lf_cfg = pst->npc->prx_lfcfg[i].i; 236 if (lf_cfg == lid) { 237 for (j = 0; j < NPC_MAX_LFL; j++) { 238 lfinfo = pst->npc->prx_fxcfg[intf][i][j].xtract; 239 rc = npc_update_extraction_data(pst, info, 240 &lfinfo[0]); 241 if (rc != 0) 242 return rc; 243 244 if (lfinfo[0].enable) 245 pst->flags[lid] = j; 246 } 247 } 248 } 249 250 done: 251 pst->pattern++; 252 return 0; 253 } 254 255 static int 256 npc_first_set_bit(uint64_t slab) 257 { 258 int num = 0; 259 260 if ((slab & 0xffffffff) == 0) { 261 num += 32; 262 slab >>= 32; 263 } 264 if ((slab & 0xffff) == 0) { 265 num += 16; 266 slab >>= 16; 267 } 268 if ((slab & 0xff) == 0) { 269 num += 8; 270 slab >>= 8; 271 } 272 if ((slab & 0xf) == 0) { 273 num += 4; 274 slab >>= 4; 275 } 276 if ((slab & 0x3) == 0) { 277 num += 2; 278 slab >>= 2; 279 } 280 if ((slab & 0x1) == 0) 281 num += 1; 282 283 return num; 284 } 285 286 static int 287 npc_shift_lv_ent(struct mbox *mbox, struct roc_npc_flow *flow, struct npc *npc, 288 uint32_t old_ent, uint32_t new_ent) 289 { 290 struct npc_mcam_shift_entry_req *req; 291 struct npc_mcam_shift_entry_rsp *rsp; 292 struct npc_flow_list *list; 293 struct roc_npc_flow *flow_iter; 294 int rc = -ENOSPC; 295 296 list = &npc->flow_list[flow->priority]; 297 298 /* Old entry is disabled & it's contents are moved to new_entry, 299 * new entry is enabled finally. 300 */ 301 req = mbox_alloc_msg_npc_mcam_shift_entry(mbox); 302 if (req == NULL) 303 return rc; 304 req->curr_entry[0] = old_ent; 305 req->new_entry[0] = new_ent; 306 req->shift_count = 1; 307 308 rc = mbox_process_msg(mbox, (void *)&rsp); 309 if (rc) 310 return rc; 311 312 /* Remove old node from list */ 313 TAILQ_FOREACH(flow_iter, list, next) { 314 if (flow_iter->mcam_id == old_ent) 315 TAILQ_REMOVE(list, flow_iter, next); 316 } 317 318 /* Insert node with new mcam id at right place */ 319 TAILQ_FOREACH(flow_iter, list, next) { 320 if (flow_iter->mcam_id > new_ent) 321 TAILQ_INSERT_BEFORE(flow_iter, flow, next); 322 } 323 return rc; 324 } 325 326 /* Exchange all required entries with a given priority level */ 327 static int 328 npc_shift_ent(struct mbox *mbox, struct roc_npc_flow *flow, struct npc *npc, 329 struct npc_mcam_alloc_entry_rsp *rsp, int dir, int prio_lvl) 330 { 331 struct plt_bitmap *fr_bmp, *fr_bmp_rev, *lv_bmp, *lv_bmp_rev, *bmp; 332 uint32_t e_fr = 0, e_lv = 0, e, e_id = 0, mcam_entries; 333 uint64_t fr_bit_pos = 0, lv_bit_pos = 0, bit_pos = 0; 334 /* Bit position within the slab */ 335 uint32_t sl_fr_bit_off = 0, sl_lv_bit_off = 0; 336 /* Overall bit position of the start of slab */ 337 /* free & live entry index */ 338 int rc_fr = 0, rc_lv = 0, rc = 0, idx = 0; 339 struct npc_mcam_ents_info *ent_info; 340 /* free & live bitmap slab */ 341 uint64_t sl_fr = 0, sl_lv = 0, *sl; 342 343 fr_bmp = npc->free_entries[prio_lvl]; 344 fr_bmp_rev = npc->free_entries_rev[prio_lvl]; 345 lv_bmp = npc->live_entries[prio_lvl]; 346 lv_bmp_rev = npc->live_entries_rev[prio_lvl]; 347 ent_info = &npc->flow_entry_info[prio_lvl]; 348 mcam_entries = npc->mcam_entries; 349 350 /* New entries allocated are always contiguous, but older entries 351 * already in free/live bitmap can be non-contiguous: so return 352 * shifted entries should be in non-contiguous format. 353 */ 354 while (idx <= rsp->count) { 355 if (!sl_fr && !sl_lv) { 356 /* Lower index elements to be exchanged */ 357 if (dir < 0) { 358 rc_fr = plt_bitmap_scan(fr_bmp, &e_fr, &sl_fr); 359 rc_lv = plt_bitmap_scan(lv_bmp, &e_lv, &sl_lv); 360 } else { 361 rc_fr = plt_bitmap_scan(fr_bmp_rev, 362 &sl_fr_bit_off, &sl_fr); 363 rc_lv = plt_bitmap_scan(lv_bmp_rev, 364 &sl_lv_bit_off, &sl_lv); 365 } 366 } 367 368 if (rc_fr) { 369 fr_bit_pos = npc_first_set_bit(sl_fr); 370 e_fr = sl_fr_bit_off + fr_bit_pos; 371 } else { 372 e_fr = ~(0); 373 } 374 375 if (rc_lv) { 376 lv_bit_pos = npc_first_set_bit(sl_lv); 377 e_lv = sl_lv_bit_off + lv_bit_pos; 378 } else { 379 e_lv = ~(0); 380 } 381 382 /* First entry is from free_bmap */ 383 if (e_fr < e_lv) { 384 bmp = fr_bmp; 385 e = e_fr; 386 sl = &sl_fr; 387 bit_pos = fr_bit_pos; 388 if (dir > 0) 389 e_id = mcam_entries - e - 1; 390 else 391 e_id = e; 392 } else { 393 bmp = lv_bmp; 394 e = e_lv; 395 sl = &sl_lv; 396 bit_pos = lv_bit_pos; 397 if (dir > 0) 398 e_id = mcam_entries - e - 1; 399 else 400 e_id = e; 401 402 if (idx < rsp->count) 403 rc = npc_shift_lv_ent(mbox, flow, npc, e_id, 404 rsp->entry + idx); 405 } 406 407 plt_bitmap_clear(bmp, e); 408 plt_bitmap_set(bmp, rsp->entry + idx); 409 /* Update entry list, use non-contiguous 410 * list now. 411 */ 412 rsp->entry_list[idx] = e_id; 413 *sl &= ~(1UL << bit_pos); 414 415 /* Update min & max entry identifiers in current 416 * priority level. 417 */ 418 if (dir < 0) { 419 ent_info->max_id = rsp->entry + idx; 420 ent_info->min_id = e_id; 421 } else { 422 ent_info->max_id = e_id; 423 ent_info->min_id = rsp->entry; 424 } 425 426 idx++; 427 } 428 return rc; 429 } 430 431 /* Validate if newly allocated entries lie in the correct priority zone 432 * since NPC_MCAM_LOWER_PRIO & NPC_MCAM_HIGHER_PRIO don't ensure zone accuracy. 433 * If not properly aligned, shift entries to do so 434 */ 435 static int 436 npc_validate_and_shift_prio_ent(struct mbox *mbox, struct roc_npc_flow *flow, 437 struct npc *npc, 438 struct npc_mcam_alloc_entry_rsp *rsp, 439 int req_prio) 440 { 441 int prio_idx = 0, rc = 0, needs_shift = 0, idx, prio = flow->priority; 442 struct npc_mcam_ents_info *info = npc->flow_entry_info; 443 int dir = (req_prio == NPC_MCAM_HIGHER_PRIO) ? 1 : -1; 444 uint32_t tot_ent = 0; 445 446 if (dir < 0) 447 prio_idx = npc->flow_max_priority - 1; 448 449 /* Only live entries needs to be shifted, free entries can just be 450 * moved by bits manipulation. 451 */ 452 453 /* For dir = -1(NPC_MCAM_LOWER_PRIO), when shifting, 454 * NPC_MAX_PREALLOC_ENT are exchanged with adjoining higher priority 455 * level entries(lower indexes). 456 * 457 * For dir = +1(NPC_MCAM_HIGHER_PRIO), during shift, 458 * NPC_MAX_PREALLOC_ENT are exchanged with adjoining lower priority 459 * level entries(higher indexes) with highest indexes. 460 */ 461 do { 462 tot_ent = info[prio_idx].free_ent + info[prio_idx].live_ent; 463 464 if (dir < 0 && prio_idx != prio && 465 rsp->entry > info[prio_idx].max_id && tot_ent) { 466 needs_shift = 1; 467 } else if ((dir > 0) && (prio_idx != prio) && 468 (rsp->entry < info[prio_idx].min_id) && tot_ent) { 469 needs_shift = 1; 470 } 471 472 if (needs_shift) { 473 needs_shift = 0; 474 rc = npc_shift_ent(mbox, flow, npc, rsp, dir, prio_idx); 475 } else { 476 for (idx = 0; idx < rsp->count; idx++) 477 rsp->entry_list[idx] = rsp->entry + idx; 478 } 479 } while ((prio_idx != prio) && (prio_idx += dir)); 480 481 return rc; 482 } 483 484 static int 485 npc_find_ref_entry(struct npc *npc, int *prio, int prio_lvl) 486 { 487 struct npc_mcam_ents_info *info = npc->flow_entry_info; 488 int step = 1; 489 490 while (step < npc->flow_max_priority) { 491 if (((prio_lvl + step) < npc->flow_max_priority) && 492 info[prio_lvl + step].live_ent) { 493 *prio = NPC_MCAM_HIGHER_PRIO; 494 return info[prio_lvl + step].min_id; 495 } 496 497 if (((prio_lvl - step) >= 0) && 498 info[prio_lvl - step].live_ent) { 499 *prio = NPC_MCAM_LOWER_PRIO; 500 return info[prio_lvl - step].max_id; 501 } 502 step++; 503 } 504 *prio = NPC_MCAM_ANY_PRIO; 505 return 0; 506 } 507 508 static int 509 npc_fill_entry_cache(struct mbox *mbox, struct roc_npc_flow *flow, 510 struct npc *npc, uint32_t *free_ent) 511 { 512 struct plt_bitmap *free_bmp, *free_bmp_rev, *live_bmp, *live_bmp_rev; 513 struct npc_mcam_alloc_entry_rsp rsp_local; 514 struct npc_mcam_alloc_entry_rsp *rsp_cmd; 515 struct npc_mcam_alloc_entry_req *req; 516 struct npc_mcam_alloc_entry_rsp *rsp; 517 struct npc_mcam_ents_info *info; 518 int rc = -ENOSPC, prio; 519 uint16_t ref_ent, idx; 520 521 info = &npc->flow_entry_info[flow->priority]; 522 free_bmp = npc->free_entries[flow->priority]; 523 free_bmp_rev = npc->free_entries_rev[flow->priority]; 524 live_bmp = npc->live_entries[flow->priority]; 525 live_bmp_rev = npc->live_entries_rev[flow->priority]; 526 527 ref_ent = npc_find_ref_entry(npc, &prio, flow->priority); 528 529 req = mbox_alloc_msg_npc_mcam_alloc_entry(mbox); 530 if (req == NULL) 531 return rc; 532 req->contig = 1; 533 req->count = npc->flow_prealloc_size; 534 req->priority = prio; 535 req->ref_entry = ref_ent; 536 537 rc = mbox_process_msg(mbox, (void *)&rsp_cmd); 538 if (rc) 539 return rc; 540 541 rsp = &rsp_local; 542 memcpy(rsp, rsp_cmd, sizeof(*rsp)); 543 544 /* Non-first ent cache fill */ 545 if (prio != NPC_MCAM_ANY_PRIO) { 546 npc_validate_and_shift_prio_ent(mbox, flow, npc, rsp, prio); 547 } else { 548 /* Copy into response entry list */ 549 for (idx = 0; idx < rsp->count; idx++) 550 rsp->entry_list[idx] = rsp->entry + idx; 551 } 552 553 /* Update free entries, reverse free entries list, 554 * min & max entry ids. 555 */ 556 for (idx = 0; idx < rsp->count; idx++) { 557 if (unlikely(rsp->entry_list[idx] < info->min_id)) 558 info->min_id = rsp->entry_list[idx]; 559 560 if (unlikely(rsp->entry_list[idx] > info->max_id)) 561 info->max_id = rsp->entry_list[idx]; 562 563 /* Skip entry to be returned, not to be part of free 564 * list. 565 */ 566 if (prio == NPC_MCAM_HIGHER_PRIO) { 567 if (unlikely(idx == (rsp->count - 1))) { 568 *free_ent = rsp->entry_list[idx]; 569 continue; 570 } 571 } else { 572 if (unlikely(!idx)) { 573 *free_ent = rsp->entry_list[idx]; 574 continue; 575 } 576 } 577 info->free_ent++; 578 plt_bitmap_set(free_bmp, rsp->entry_list[idx]); 579 plt_bitmap_set(free_bmp_rev, 580 npc->mcam_entries - rsp->entry_list[idx] - 1); 581 } 582 583 info->live_ent++; 584 plt_bitmap_set(live_bmp, *free_ent); 585 plt_bitmap_set(live_bmp_rev, npc->mcam_entries - *free_ent - 1); 586 587 return 0; 588 } 589 590 int 591 npc_check_preallocated_entry_cache(struct mbox *mbox, struct roc_npc_flow *flow, 592 struct npc *npc) 593 { 594 struct plt_bitmap *free, *free_rev, *live, *live_rev; 595 uint32_t pos = 0, free_ent = 0, mcam_entries; 596 struct npc_mcam_ents_info *info; 597 uint64_t slab = 0; 598 int rc; 599 600 info = &npc->flow_entry_info[flow->priority]; 601 602 free_rev = npc->free_entries_rev[flow->priority]; 603 free = npc->free_entries[flow->priority]; 604 live_rev = npc->live_entries_rev[flow->priority]; 605 live = npc->live_entries[flow->priority]; 606 mcam_entries = npc->mcam_entries; 607 608 if (info->free_ent) { 609 rc = plt_bitmap_scan(free, &pos, &slab); 610 if (rc) { 611 /* Get free_ent from free entry bitmap */ 612 free_ent = pos + __builtin_ctzll(slab); 613 /* Remove from free bitmaps and add to live ones */ 614 plt_bitmap_clear(free, free_ent); 615 plt_bitmap_set(live, free_ent); 616 plt_bitmap_clear(free_rev, mcam_entries - free_ent - 1); 617 plt_bitmap_set(live_rev, mcam_entries - free_ent - 1); 618 619 info->free_ent--; 620 info->live_ent++; 621 return free_ent; 622 } 623 return NPC_ERR_INTERNAL; 624 } 625 626 rc = npc_fill_entry_cache(mbox, flow, npc, &free_ent); 627 if (rc) 628 return rc; 629 630 return free_ent; 631 } 632