1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 #include "roc_api.h" 5 #include "roc_priv.h" 6 7 static void 8 npc_prep_mcam_ldata(uint8_t *ptr, const uint8_t *data, int len) 9 { 10 int idx; 11 12 for (idx = 0; idx < len; idx++) 13 ptr[idx] = data[len - 1 - idx]; 14 } 15 16 static int 17 npc_check_copysz(size_t size, size_t len) 18 { 19 if (len <= size) 20 return len; 21 return NPC_ERR_PARAM; 22 } 23 24 static inline int 25 npc_mem_is_zero(const void *mem, int len) 26 { 27 const char *m = mem; 28 int i; 29 30 for (i = 0; i < len; i++) { 31 if (m[i] != 0) 32 return 0; 33 } 34 return 1; 35 } 36 37 static void 38 npc_set_hw_mask(struct npc_parse_item_info *info, struct npc_xtract_info *xinfo, 39 char *hw_mask) 40 { 41 int max_off, offset; 42 int j; 43 44 if (xinfo->enable == 0) 45 return; 46 47 if (xinfo->hdr_off < info->hw_hdr_len) 48 return; 49 50 max_off = xinfo->hdr_off + xinfo->len - info->hw_hdr_len; 51 52 if (max_off > info->len) 53 max_off = info->len; 54 55 offset = xinfo->hdr_off - info->hw_hdr_len; 56 for (j = offset; j < max_off; j++) 57 hw_mask[j] = 0xff; 58 } 59 60 void 61 npc_get_hw_supp_mask(struct npc_parse_state *pst, 62 struct npc_parse_item_info *info, int lid, int lt) 63 { 64 struct npc_xtract_info *xinfo, *lfinfo; 65 char *hw_mask = info->hw_mask; 66 int lf_cfg = 0; 67 int i, j; 68 int intf; 69 70 intf = pst->nix_intf; 71 xinfo = pst->npc->prx_dxcfg[intf][lid][lt].xtract; 72 memset(hw_mask, 0, info->len); 73 74 for (i = 0; i < NPC_MAX_LD; i++) 75 npc_set_hw_mask(info, &xinfo[i], hw_mask); 76 77 for (i = 0; i < NPC_MAX_LD; i++) { 78 if (xinfo[i].flags_enable == 0) 79 continue; 80 81 lf_cfg = pst->npc->prx_lfcfg[i].i; 82 if (lf_cfg == lid) { 83 for (j = 0; j < NPC_MAX_LFL; j++) { 84 lfinfo = pst->npc->prx_fxcfg[intf][i][j].xtract; 85 npc_set_hw_mask(info, &lfinfo[0], hw_mask); 86 } 87 } 88 } 89 } 90 91 static inline int 92 npc_mask_is_supported(const char *mask, const char *hw_mask, int len) 93 { 94 /* 95 * If no hw_mask, assume nothing is supported. 96 * mask is never NULL 97 */ 98 if (hw_mask == NULL) 99 return npc_mem_is_zero(mask, len); 100 101 while (len--) { 102 if ((mask[len] | hw_mask[len]) != hw_mask[len]) 103 return 0; /* False */ 104 } 105 return 1; 106 } 107 108 int 109 npc_parse_item_basic(const struct roc_npc_item_info *item, 110 struct npc_parse_item_info *info) 111 { 112 /* Item must not be NULL */ 113 if (item == NULL) 114 return NPC_ERR_PARAM; 115 116 /* Don't support ranges */ 117 if (item->last != NULL) 118 return NPC_ERR_INVALID_RANGE; 119 120 /* If spec is NULL, both mask and last must be NULL, this 121 * makes it to match ANY value (eq to mask = 0). 122 * Setting either mask or last without spec is an error 123 */ 124 if (item->spec == NULL) { 125 if (item->last == NULL && item->mask == NULL) { 126 info->spec = NULL; 127 return 0; 128 } 129 return NPC_ERR_INVALID_SPEC; 130 } 131 132 /* We have valid spec */ 133 info->spec = item->spec; 134 135 /* If mask is not set, use default mask, err if default mask is 136 * also NULL. 137 */ 138 if (item->mask == NULL) { 139 if (info->def_mask == NULL) 140 return NPC_ERR_PARAM; 141 info->mask = info->def_mask; 142 } else { 143 info->mask = item->mask; 144 } 145 146 /* mask specified must be subset of hw supported mask 147 * mask | hw_mask == hw_mask 148 */ 149 if (!npc_mask_is_supported(info->mask, info->hw_mask, info->len)) 150 return NPC_ERR_INVALID_MASK; 151 152 return 0; 153 } 154 155 static int 156 npc_update_extraction_data(struct npc_parse_state *pst, 157 struct npc_parse_item_info *info, 158 struct npc_xtract_info *xinfo) 159 { 160 uint8_t int_info_mask[NPC_MAX_EXTRACT_DATA_LEN]; 161 uint8_t int_info[NPC_MAX_EXTRACT_DATA_LEN]; 162 struct npc_xtract_info *x; 163 int hdr_off; 164 int len = 0; 165 166 x = xinfo; 167 len = x->len; 168 hdr_off = x->hdr_off; 169 170 if (hdr_off < info->hw_hdr_len) 171 return 0; 172 173 if (x->enable == 0) 174 return 0; 175 176 hdr_off -= info->hw_hdr_len; 177 178 if (hdr_off >= info->len) 179 return 0; 180 181 if (hdr_off + len > info->len) 182 len = info->len - hdr_off; 183 184 len = npc_check_copysz((ROC_NPC_MAX_MCAM_WIDTH_DWORDS * 8) - x->key_off, 185 len); 186 if (len < 0) 187 return NPC_ERR_INVALID_SIZE; 188 189 /* Need to reverse complete structure so that dest addr is at 190 * MSB so as to program the MCAM using mcam_data & mcam_mask 191 * arrays 192 */ 193 npc_prep_mcam_ldata(int_info, (const uint8_t *)info->spec + hdr_off, 194 x->len); 195 npc_prep_mcam_ldata(int_info_mask, 196 (const uint8_t *)info->mask + hdr_off, x->len); 197 198 memcpy(pst->mcam_mask + x->key_off, int_info_mask, len); 199 memcpy(pst->mcam_data + x->key_off, int_info, len); 200 return 0; 201 } 202 203 int 204 npc_update_parse_state(struct npc_parse_state *pst, 205 struct npc_parse_item_info *info, int lid, int lt, 206 uint8_t flags) 207 { 208 struct npc_lid_lt_xtract_info *xinfo; 209 struct roc_npc_flow_dump_data *dump; 210 struct npc_xtract_info *lfinfo; 211 int intf, lf_cfg; 212 int i, j, rc = 0; 213 214 pst->layer_mask |= lid; 215 pst->lt[lid] = lt; 216 pst->flags[lid] = flags; 217 218 intf = pst->nix_intf; 219 xinfo = &pst->npc->prx_dxcfg[intf][lid][lt]; 220 if (xinfo->is_terminating) 221 pst->terminate = 1; 222 223 if (info->spec == NULL) 224 goto done; 225 226 for (i = 0; i < NPC_MAX_LD; i++) { 227 rc = npc_update_extraction_data(pst, info, &xinfo->xtract[i]); 228 if (rc != 0) 229 return rc; 230 } 231 232 for (i = 0; i < NPC_MAX_LD; i++) { 233 if (xinfo->xtract[i].flags_enable == 0) 234 continue; 235 236 lf_cfg = pst->npc->prx_lfcfg[i].i; 237 if (lf_cfg == lid) { 238 for (j = 0; j < NPC_MAX_LFL; j++) { 239 lfinfo = pst->npc->prx_fxcfg[intf][i][j].xtract; 240 rc = npc_update_extraction_data(pst, info, 241 &lfinfo[0]); 242 if (rc != 0) 243 return rc; 244 245 if (lfinfo[0].enable) 246 pst->flags[lid] = j; 247 } 248 } 249 } 250 251 done: 252 dump = &pst->flow->dump_data[pst->flow->num_patterns++]; 253 dump->lid = lid; 254 dump->ltype = lt; 255 pst->pattern++; 256 return 0; 257 } 258 259 static int 260 npc_first_set_bit(uint64_t slab) 261 { 262 int num = 0; 263 264 if ((slab & 0xffffffff) == 0) { 265 num += 32; 266 slab >>= 32; 267 } 268 if ((slab & 0xffff) == 0) { 269 num += 16; 270 slab >>= 16; 271 } 272 if ((slab & 0xff) == 0) { 273 num += 8; 274 slab >>= 8; 275 } 276 if ((slab & 0xf) == 0) { 277 num += 4; 278 slab >>= 4; 279 } 280 if ((slab & 0x3) == 0) { 281 num += 2; 282 slab >>= 2; 283 } 284 if ((slab & 0x1) == 0) 285 num += 1; 286 287 return num; 288 } 289 290 static int 291 npc_shift_lv_ent(struct mbox *mbox, struct roc_npc_flow *flow, struct npc *npc, 292 uint32_t old_ent, uint32_t new_ent) 293 { 294 struct npc_mcam_shift_entry_req *req; 295 struct npc_mcam_shift_entry_rsp *rsp; 296 struct npc_flow_list *list; 297 struct roc_npc_flow *flow_iter; 298 int rc = -ENOSPC; 299 300 list = &npc->flow_list[flow->priority]; 301 302 /* Old entry is disabled & it's contents are moved to new_entry, 303 * new entry is enabled finally. 304 */ 305 req = mbox_alloc_msg_npc_mcam_shift_entry(mbox); 306 if (req == NULL) 307 return rc; 308 req->curr_entry[0] = old_ent; 309 req->new_entry[0] = new_ent; 310 req->shift_count = 1; 311 312 rc = mbox_process_msg(mbox, (void *)&rsp); 313 if (rc) 314 return rc; 315 316 /* Remove old node from list */ 317 TAILQ_FOREACH(flow_iter, list, next) { 318 if (flow_iter->mcam_id == old_ent) 319 TAILQ_REMOVE(list, flow_iter, next); 320 } 321 322 /* Insert node with new mcam id at right place */ 323 TAILQ_FOREACH(flow_iter, list, next) { 324 if (flow_iter->mcam_id > new_ent) 325 TAILQ_INSERT_BEFORE(flow_iter, flow, next); 326 } 327 return rc; 328 } 329 330 /* Exchange all required entries with a given priority level */ 331 static int 332 npc_shift_ent(struct mbox *mbox, struct roc_npc_flow *flow, struct npc *npc, 333 struct npc_mcam_alloc_entry_rsp *rsp, int dir, int prio_lvl) 334 { 335 struct plt_bitmap *fr_bmp, *fr_bmp_rev, *lv_bmp, *lv_bmp_rev, *bmp; 336 uint32_t e_fr = 0, e_lv = 0, e, e_id = 0, mcam_entries; 337 uint64_t fr_bit_pos = 0, lv_bit_pos = 0, bit_pos = 0; 338 /* Bit position within the slab */ 339 uint32_t sl_fr_bit_off = 0, sl_lv_bit_off = 0; 340 /* Overall bit position of the start of slab */ 341 /* free & live entry index */ 342 int rc_fr = 0, rc_lv = 0, rc = 0, idx = 0; 343 struct npc_mcam_ents_info *ent_info; 344 /* free & live bitmap slab */ 345 uint64_t sl_fr = 0, sl_lv = 0, *sl; 346 347 fr_bmp = npc->free_entries[prio_lvl]; 348 fr_bmp_rev = npc->free_entries_rev[prio_lvl]; 349 lv_bmp = npc->live_entries[prio_lvl]; 350 lv_bmp_rev = npc->live_entries_rev[prio_lvl]; 351 ent_info = &npc->flow_entry_info[prio_lvl]; 352 mcam_entries = npc->mcam_entries; 353 354 /* New entries allocated are always contiguous, but older entries 355 * already in free/live bitmap can be non-contiguous: so return 356 * shifted entries should be in non-contiguous format. 357 */ 358 while (idx <= rsp->count) { 359 if (!sl_fr && !sl_lv) { 360 /* Lower index elements to be exchanged */ 361 if (dir < 0) { 362 rc_fr = plt_bitmap_scan(fr_bmp, &e_fr, &sl_fr); 363 rc_lv = plt_bitmap_scan(lv_bmp, &e_lv, &sl_lv); 364 } else { 365 rc_fr = plt_bitmap_scan(fr_bmp_rev, 366 &sl_fr_bit_off, &sl_fr); 367 rc_lv = plt_bitmap_scan(lv_bmp_rev, 368 &sl_lv_bit_off, &sl_lv); 369 } 370 } 371 372 if (rc_fr) { 373 fr_bit_pos = npc_first_set_bit(sl_fr); 374 e_fr = sl_fr_bit_off + fr_bit_pos; 375 } else { 376 e_fr = ~(0); 377 } 378 379 if (rc_lv) { 380 lv_bit_pos = npc_first_set_bit(sl_lv); 381 e_lv = sl_lv_bit_off + lv_bit_pos; 382 } else { 383 e_lv = ~(0); 384 } 385 386 /* First entry is from free_bmap */ 387 if (e_fr < e_lv) { 388 bmp = fr_bmp; 389 e = e_fr; 390 sl = &sl_fr; 391 bit_pos = fr_bit_pos; 392 if (dir > 0) 393 e_id = mcam_entries - e - 1; 394 else 395 e_id = e; 396 } else { 397 bmp = lv_bmp; 398 e = e_lv; 399 sl = &sl_lv; 400 bit_pos = lv_bit_pos; 401 if (dir > 0) 402 e_id = mcam_entries - e - 1; 403 else 404 e_id = e; 405 406 if (idx < rsp->count) 407 rc = npc_shift_lv_ent(mbox, flow, npc, e_id, 408 rsp->entry + idx); 409 } 410 411 plt_bitmap_clear(bmp, e); 412 plt_bitmap_set(bmp, rsp->entry + idx); 413 /* Update entry list, use non-contiguous 414 * list now. 415 */ 416 rsp->entry_list[idx] = e_id; 417 *sl &= ~(1UL << bit_pos); 418 419 /* Update min & max entry identifiers in current 420 * priority level. 421 */ 422 if (dir < 0) { 423 ent_info->max_id = rsp->entry + idx; 424 ent_info->min_id = e_id; 425 } else { 426 ent_info->max_id = e_id; 427 ent_info->min_id = rsp->entry; 428 } 429 430 idx++; 431 } 432 return rc; 433 } 434 435 /* Validate if newly allocated entries lie in the correct priority zone 436 * since NPC_MCAM_LOWER_PRIO & NPC_MCAM_HIGHER_PRIO don't ensure zone accuracy. 437 * If not properly aligned, shift entries to do so 438 */ 439 static int 440 npc_validate_and_shift_prio_ent(struct mbox *mbox, struct roc_npc_flow *flow, 441 struct npc *npc, 442 struct npc_mcam_alloc_entry_rsp *rsp, 443 int req_prio) 444 { 445 int prio_idx = 0, rc = 0, needs_shift = 0, idx, prio = flow->priority; 446 struct npc_mcam_ents_info *info = npc->flow_entry_info; 447 int dir = (req_prio == NPC_MCAM_HIGHER_PRIO) ? 1 : -1; 448 uint32_t tot_ent = 0; 449 450 if (dir < 0) 451 prio_idx = npc->flow_max_priority - 1; 452 453 /* Only live entries needs to be shifted, free entries can just be 454 * moved by bits manipulation. 455 */ 456 457 /* For dir = -1(NPC_MCAM_LOWER_PRIO), when shifting, 458 * NPC_MAX_PREALLOC_ENT are exchanged with adjoining higher priority 459 * level entries(lower indexes). 460 * 461 * For dir = +1(NPC_MCAM_HIGHER_PRIO), during shift, 462 * NPC_MAX_PREALLOC_ENT are exchanged with adjoining lower priority 463 * level entries(higher indexes) with highest indexes. 464 */ 465 do { 466 tot_ent = info[prio_idx].free_ent + info[prio_idx].live_ent; 467 468 if (dir < 0 && prio_idx != prio && 469 rsp->entry > info[prio_idx].max_id && tot_ent) { 470 needs_shift = 1; 471 } else if ((dir > 0) && (prio_idx != prio) && 472 (rsp->entry < info[prio_idx].min_id) && tot_ent) { 473 needs_shift = 1; 474 } 475 476 if (needs_shift) { 477 needs_shift = 0; 478 rc = npc_shift_ent(mbox, flow, npc, rsp, dir, prio_idx); 479 } else { 480 for (idx = 0; idx < rsp->count; idx++) 481 rsp->entry_list[idx] = rsp->entry + idx; 482 } 483 } while ((prio_idx != prio) && (prio_idx += dir)); 484 485 return rc; 486 } 487 488 static int 489 npc_find_ref_entry(struct npc *npc, int *prio, int prio_lvl) 490 { 491 struct npc_mcam_ents_info *info = npc->flow_entry_info; 492 int step = 1; 493 494 while (step < npc->flow_max_priority) { 495 if (((prio_lvl + step) < npc->flow_max_priority) && 496 info[prio_lvl + step].live_ent) { 497 *prio = NPC_MCAM_HIGHER_PRIO; 498 return info[prio_lvl + step].min_id; 499 } 500 501 if (((prio_lvl - step) >= 0) && 502 info[prio_lvl - step].live_ent) { 503 *prio = NPC_MCAM_LOWER_PRIO; 504 return info[prio_lvl - step].max_id; 505 } 506 step++; 507 } 508 *prio = NPC_MCAM_ANY_PRIO; 509 return 0; 510 } 511 512 static int 513 npc_fill_entry_cache(struct mbox *mbox, struct roc_npc_flow *flow, 514 struct npc *npc, uint32_t *free_ent) 515 { 516 struct plt_bitmap *free_bmp, *free_bmp_rev, *live_bmp, *live_bmp_rev; 517 struct npc_mcam_alloc_entry_rsp rsp_local; 518 struct npc_mcam_alloc_entry_rsp *rsp_cmd; 519 struct npc_mcam_alloc_entry_req *req; 520 struct npc_mcam_alloc_entry_rsp *rsp; 521 struct npc_mcam_ents_info *info; 522 int rc = -ENOSPC, prio; 523 uint16_t ref_ent, idx; 524 525 info = &npc->flow_entry_info[flow->priority]; 526 free_bmp = npc->free_entries[flow->priority]; 527 free_bmp_rev = npc->free_entries_rev[flow->priority]; 528 live_bmp = npc->live_entries[flow->priority]; 529 live_bmp_rev = npc->live_entries_rev[flow->priority]; 530 531 ref_ent = npc_find_ref_entry(npc, &prio, flow->priority); 532 533 req = mbox_alloc_msg_npc_mcam_alloc_entry(mbox); 534 if (req == NULL) 535 return rc; 536 req->contig = 1; 537 req->count = npc->flow_prealloc_size; 538 req->priority = prio; 539 req->ref_entry = ref_ent; 540 541 rc = mbox_process_msg(mbox, (void *)&rsp_cmd); 542 if (rc) 543 return rc; 544 545 rsp = &rsp_local; 546 memcpy(rsp, rsp_cmd, sizeof(*rsp)); 547 548 /* Non-first ent cache fill */ 549 if (prio != NPC_MCAM_ANY_PRIO) { 550 npc_validate_and_shift_prio_ent(mbox, flow, npc, rsp, prio); 551 } else { 552 /* Copy into response entry list */ 553 for (idx = 0; idx < rsp->count; idx++) 554 rsp->entry_list[idx] = rsp->entry + idx; 555 } 556 557 /* Update free entries, reverse free entries list, 558 * min & max entry ids. 559 */ 560 for (idx = 0; idx < rsp->count; idx++) { 561 if (unlikely(rsp->entry_list[idx] < info->min_id)) 562 info->min_id = rsp->entry_list[idx]; 563 564 if (unlikely(rsp->entry_list[idx] > info->max_id)) 565 info->max_id = rsp->entry_list[idx]; 566 567 /* Skip entry to be returned, not to be part of free 568 * list. 569 */ 570 if (prio == NPC_MCAM_HIGHER_PRIO) { 571 if (unlikely(idx == (rsp->count - 1))) { 572 *free_ent = rsp->entry_list[idx]; 573 continue; 574 } 575 } else { 576 if (unlikely(!idx)) { 577 *free_ent = rsp->entry_list[idx]; 578 continue; 579 } 580 } 581 info->free_ent++; 582 plt_bitmap_set(free_bmp, rsp->entry_list[idx]); 583 plt_bitmap_set(free_bmp_rev, 584 npc->mcam_entries - rsp->entry_list[idx] - 1); 585 } 586 587 info->live_ent++; 588 plt_bitmap_set(live_bmp, *free_ent); 589 plt_bitmap_set(live_bmp_rev, npc->mcam_entries - *free_ent - 1); 590 591 return 0; 592 } 593 594 int 595 npc_check_preallocated_entry_cache(struct mbox *mbox, struct roc_npc_flow *flow, 596 struct npc *npc) 597 { 598 struct plt_bitmap *free, *free_rev, *live, *live_rev; 599 uint32_t pos = 0, free_ent = 0, mcam_entries; 600 struct npc_mcam_ents_info *info; 601 uint64_t slab = 0; 602 int rc; 603 604 info = &npc->flow_entry_info[flow->priority]; 605 606 free_rev = npc->free_entries_rev[flow->priority]; 607 free = npc->free_entries[flow->priority]; 608 live_rev = npc->live_entries_rev[flow->priority]; 609 live = npc->live_entries[flow->priority]; 610 mcam_entries = npc->mcam_entries; 611 612 if (info->free_ent) { 613 rc = plt_bitmap_scan(free, &pos, &slab); 614 if (rc) { 615 /* Get free_ent from free entry bitmap */ 616 free_ent = pos + __builtin_ctzll(slab); 617 /* Remove from free bitmaps and add to live ones */ 618 plt_bitmap_clear(free, free_ent); 619 plt_bitmap_set(live, free_ent); 620 plt_bitmap_clear(free_rev, mcam_entries - free_ent - 1); 621 plt_bitmap_set(live_rev, mcam_entries - free_ent - 1); 622 623 info->free_ent--; 624 info->live_ent++; 625 return free_ent; 626 } 627 return NPC_ERR_INTERNAL; 628 } 629 630 rc = npc_fill_entry_cache(mbox, flow, npc, &free_ent); 631 if (rc) 632 return rc; 633 634 return free_ent; 635 } 636