1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019-2020 Broadcom
3 * All rights reserved.
4 */
5
6 #include <string.h>
7 #include <math.h>
8 #include <sys/param.h>
9 #include <rte_common.h>
10 #include <rte_errno.h>
11 #include <rte_log.h>
12
13 #include "tf_core.h"
14 #include "tf_util.h"
15 #include "tf_common.h"
16 #include "tf_em.h"
17 #include "tf_em_common.h"
18 #include "tf_msg.h"
19 #include "tfp.h"
20 #include "lookup3.h"
21 #include "tf_ext_flow_handle.h"
22
23 #include "bnxt.h"
24
25 #define PTU_PTE_VALID 0x1UL
26 #define PTU_PTE_LAST 0x2UL
27 #define PTU_PTE_NEXT_TO_LAST 0x4UL
28
29 /* Number of pointers per page_size */
30 #define MAX_PAGE_PTRS(page_size) ((page_size) / sizeof(void *))
31
32 /**
33 * EM DBs.
34 */
35 extern void *eem_db[TF_DIR_MAX];
36
37 extern struct tf_tbl_scope_cb tbl_scopes[TF_NUM_TBL_SCOPE];
38
39 /**
40 * Function to free a page table
41 *
42 * [in] tp
43 * Pointer to the page table to free
44 */
45 static void
tf_em_free_pg_tbl(struct hcapi_cfa_em_page_tbl * tp)46 tf_em_free_pg_tbl(struct hcapi_cfa_em_page_tbl *tp)
47 {
48 uint32_t i;
49
50 for (i = 0; i < tp->pg_count; i++) {
51 if (!tp->pg_va_tbl[i]) {
52 TFP_DRV_LOG(WARNING,
53 "No mapping for page: %d table: %016" PRIu64 "\n",
54 i,
55 (uint64_t)(uintptr_t)tp);
56 continue;
57 }
58
59 tfp_free(tp->pg_va_tbl[i]);
60 tp->pg_va_tbl[i] = NULL;
61 }
62
63 tp->pg_count = 0;
64 tfp_free(tp->pg_va_tbl);
65 tp->pg_va_tbl = NULL;
66 tfp_free(tp->pg_pa_tbl);
67 tp->pg_pa_tbl = NULL;
68 }
69
70 /**
71 * Function to free an EM table
72 *
73 * [in] tbl
74 * Pointer to the EM table to free
75 */
76 static void
tf_em_free_page_table(struct hcapi_cfa_em_table * tbl)77 tf_em_free_page_table(struct hcapi_cfa_em_table *tbl)
78 {
79 struct hcapi_cfa_em_page_tbl *tp;
80 int i;
81
82 for (i = 0; i < tbl->num_lvl; i++) {
83 tp = &tbl->pg_tbl[i];
84 TFP_DRV_LOG(INFO,
85 "EEM: Freeing page table: size %u lvl %d cnt %u\n",
86 TF_EM_PAGE_SIZE,
87 i,
88 tp->pg_count);
89
90 tf_em_free_pg_tbl(tp);
91 }
92
93 tbl->l0_addr = NULL;
94 tbl->l0_dma_addr = 0;
95 tbl->num_lvl = 0;
96 tbl->num_data_pages = 0;
97 }
98
99 /**
100 * Allocation of page tables
101 *
102 * [in] tfp
103 * Pointer to a TruFlow handle
104 *
105 * [in] pg_count
106 * Page count to allocate
107 *
108 * [in] pg_size
109 * Size of each page
110 *
111 * Returns:
112 * 0 - Success
113 * -ENOMEM - Out of memory
114 */
115 static int
tf_em_alloc_pg_tbl(struct hcapi_cfa_em_page_tbl * tp,uint32_t pg_count,uint32_t pg_size)116 tf_em_alloc_pg_tbl(struct hcapi_cfa_em_page_tbl *tp,
117 uint32_t pg_count,
118 uint32_t pg_size)
119 {
120 uint32_t i;
121 struct tfp_calloc_parms parms;
122
123 parms.nitems = pg_count;
124 parms.size = sizeof(void *);
125 parms.alignment = 0;
126
127 if (tfp_calloc(&parms) != 0)
128 return -ENOMEM;
129
130 tp->pg_va_tbl = parms.mem_va;
131
132 if (tfp_calloc(&parms) != 0) {
133 tfp_free(tp->pg_va_tbl);
134 return -ENOMEM;
135 }
136
137 tp->pg_pa_tbl = parms.mem_va;
138
139 tp->pg_count = 0;
140 tp->pg_size = pg_size;
141
142 for (i = 0; i < pg_count; i++) {
143 parms.nitems = 1;
144 parms.size = pg_size;
145 parms.alignment = TF_EM_PAGE_ALIGNMENT;
146
147 if (tfp_calloc(&parms) != 0)
148 goto cleanup;
149
150 tp->pg_pa_tbl[i] = (uintptr_t)parms.mem_pa;
151 tp->pg_va_tbl[i] = parms.mem_va;
152
153 memset(tp->pg_va_tbl[i], 0, pg_size);
154 tp->pg_count++;
155 }
156
157 return 0;
158
159 cleanup:
160 tf_em_free_pg_tbl(tp);
161 return -ENOMEM;
162 }
163
164 /**
165 * Allocates EM page tables
166 *
167 * [in] tbl
168 * Table to allocate pages for
169 *
170 * Returns:
171 * 0 - Success
172 * -ENOMEM - Out of memory
173 */
174 static int
tf_em_alloc_page_table(struct hcapi_cfa_em_table * tbl)175 tf_em_alloc_page_table(struct hcapi_cfa_em_table *tbl)
176 {
177 struct hcapi_cfa_em_page_tbl *tp;
178 int rc = 0;
179 int i;
180 uint32_t j;
181
182 for (i = 0; i < tbl->num_lvl; i++) {
183 tp = &tbl->pg_tbl[i];
184
185 rc = tf_em_alloc_pg_tbl(tp,
186 tbl->page_cnt[i],
187 TF_EM_PAGE_SIZE);
188 if (rc) {
189 TFP_DRV_LOG(WARNING,
190 "Failed to allocate page table: lvl: %d, rc:%s\n",
191 i,
192 strerror(-rc));
193 goto cleanup;
194 }
195
196 for (j = 0; j < tp->pg_count; j++) {
197 TFP_DRV_LOG(INFO,
198 "EEM: Allocated page table: size %u lvl %d cnt"
199 " %u VA:%p PA:%p\n",
200 TF_EM_PAGE_SIZE,
201 i,
202 tp->pg_count,
203 (void *)(uintptr_t)tp->pg_va_tbl[j],
204 (void *)(uintptr_t)tp->pg_pa_tbl[j]);
205 }
206 }
207 return rc;
208
209 cleanup:
210 tf_em_free_page_table(tbl);
211 return rc;
212 }
213
214 /**
215 * Links EM page tables
216 *
217 * [in] tp
218 * Pointer to page table
219 *
220 * [in] tp_next
221 * Pointer to the next page table
222 *
223 * [in] set_pte_last
224 * Flag controlling if the page table is last
225 */
226 static void
tf_em_link_page_table(struct hcapi_cfa_em_page_tbl * tp,struct hcapi_cfa_em_page_tbl * tp_next,bool set_pte_last)227 tf_em_link_page_table(struct hcapi_cfa_em_page_tbl *tp,
228 struct hcapi_cfa_em_page_tbl *tp_next,
229 bool set_pte_last)
230 {
231 uint64_t *pg_pa = tp_next->pg_pa_tbl;
232 uint64_t *pg_va;
233 uint64_t valid;
234 uint32_t k = 0;
235 uint32_t i;
236 uint32_t j;
237
238 for (i = 0; i < tp->pg_count; i++) {
239 pg_va = tp->pg_va_tbl[i];
240
241 for (j = 0; j < MAX_PAGE_PTRS(tp->pg_size); j++) {
242 if (k == tp_next->pg_count - 2 && set_pte_last)
243 valid = PTU_PTE_NEXT_TO_LAST | PTU_PTE_VALID;
244 else if (k == tp_next->pg_count - 1 && set_pte_last)
245 valid = PTU_PTE_LAST | PTU_PTE_VALID;
246 else
247 valid = PTU_PTE_VALID;
248
249 pg_va[j] = tfp_cpu_to_le_64(pg_pa[k] | valid);
250 if (++k >= tp_next->pg_count)
251 return;
252 }
253 }
254 }
255
256 /**
257 * Setup a EM page table
258 *
259 * [in] tbl
260 * Pointer to EM page table
261 */
262 static void
tf_em_setup_page_table(struct hcapi_cfa_em_table * tbl)263 tf_em_setup_page_table(struct hcapi_cfa_em_table *tbl)
264 {
265 struct hcapi_cfa_em_page_tbl *tp_next;
266 struct hcapi_cfa_em_page_tbl *tp;
267 bool set_pte_last = 0;
268 int i;
269
270 for (i = 0; i < tbl->num_lvl - 1; i++) {
271 tp = &tbl->pg_tbl[i];
272 tp_next = &tbl->pg_tbl[i + 1];
273 if (i == tbl->num_lvl - 2)
274 set_pte_last = 1;
275 tf_em_link_page_table(tp, tp_next, set_pte_last);
276 }
277
278 tbl->l0_addr = tbl->pg_tbl[TF_PT_LVL_0].pg_va_tbl[0];
279 tbl->l0_dma_addr = tbl->pg_tbl[TF_PT_LVL_0].pg_pa_tbl[0];
280 }
281
282 /**
283 * Unregisters EM Ctx in Firmware
284 *
285 * [in] tfp
286 * Pointer to a TruFlow handle
287 *
288 * [in] tbl_scope_cb
289 * Pointer to a table scope control block
290 *
291 * [in] dir
292 * Receive or transmit direction
293 */
294 static void
tf_em_ctx_unreg(struct tf * tfp,struct tf_tbl_scope_cb * tbl_scope_cb,int dir)295 tf_em_ctx_unreg(struct tf *tfp,
296 struct tf_tbl_scope_cb *tbl_scope_cb,
297 int dir)
298 {
299 struct hcapi_cfa_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
300 struct hcapi_cfa_em_table *tbl;
301 int i;
302
303 for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
304 tbl = &ctxp->em_tables[i];
305
306 if (tbl->num_entries != 0 && tbl->entry_size != 0) {
307 tf_msg_em_mem_unrgtr(tfp, &tbl->ctx_id);
308 tf_em_free_page_table(tbl);
309 }
310 }
311 }
312
313 /**
314 * Registers EM Ctx in Firmware
315 *
316 * [in] tfp
317 * Pointer to a TruFlow handle
318 *
319 * [in] tbl_scope_cb
320 * Pointer to a table scope control block
321 *
322 * [in] dir
323 * Receive or transmit direction
324 *
325 * Returns:
326 * 0 - Success
327 * -ENOMEM - Out of Memory
328 */
329 static int
tf_em_ctx_reg(struct tf * tfp,struct tf_tbl_scope_cb * tbl_scope_cb,int dir)330 tf_em_ctx_reg(struct tf *tfp,
331 struct tf_tbl_scope_cb *tbl_scope_cb,
332 int dir)
333 {
334 struct hcapi_cfa_em_ctx_mem_info *ctxp = &tbl_scope_cb->em_ctx_info[dir];
335 struct hcapi_cfa_em_table *tbl;
336 int rc = 0;
337 int i;
338
339 for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
340 tbl = &ctxp->em_tables[i];
341
342 if (tbl->num_entries && tbl->entry_size) {
343 rc = tf_em_size_table(tbl, TF_EM_PAGE_SIZE);
344
345 if (rc)
346 goto cleanup;
347
348 rc = tf_em_alloc_page_table(tbl);
349 if (rc)
350 goto cleanup;
351
352 tf_em_setup_page_table(tbl);
353 rc = tf_msg_em_mem_rgtr(tfp,
354 tbl->num_lvl - 1,
355 TF_EM_PAGE_SIZE_ENUM,
356 tbl->l0_dma_addr,
357 &tbl->ctx_id);
358 if (rc)
359 goto cleanup;
360 }
361 }
362 return rc;
363
364 cleanup:
365 tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
366 return rc;
367 }
368
369 int
tf_em_ext_alloc(struct tf * tfp,struct tf_alloc_tbl_scope_parms * parms)370 tf_em_ext_alloc(struct tf *tfp, struct tf_alloc_tbl_scope_parms *parms)
371 {
372 int rc;
373 enum tf_dir dir;
374 struct tf_tbl_scope_cb *tbl_scope_cb;
375 struct hcapi_cfa_em_table *em_tables;
376 struct tf_free_tbl_scope_parms free_parms;
377 struct tf_rm_allocate_parms aparms = { 0 };
378 struct tf_rm_free_parms fparms = { 0 };
379
380 /* Get Table Scope control block from the session pool */
381 aparms.rm_db = eem_db[TF_DIR_RX];
382 aparms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
383 aparms.index = (uint32_t *)&parms->tbl_scope_id;
384 rc = tf_rm_allocate(&aparms);
385 if (rc) {
386 TFP_DRV_LOG(ERR,
387 "Failed to allocate table scope\n");
388 return rc;
389 }
390
391 tbl_scope_cb = &tbl_scopes[parms->tbl_scope_id];
392 tbl_scope_cb->index = parms->tbl_scope_id;
393 tbl_scope_cb->tbl_scope_id = parms->tbl_scope_id;
394
395 rc = tfp_get_pf(tfp, &tbl_scope_cb->pf);
396 if (rc) {
397 TFP_DRV_LOG(ERR,
398 "EEM: PF query error rc:%s\n",
399 strerror(-rc));
400 goto cleanup;
401 }
402
403 for (dir = 0; dir < TF_DIR_MAX; dir++) {
404 rc = tf_msg_em_qcaps(tfp,
405 dir,
406 &tbl_scope_cb->em_caps[dir]);
407 if (rc) {
408 TFP_DRV_LOG(ERR,
409 "EEM: Unable to query for EEM capability,"
410 " rc:%s\n",
411 strerror(-rc));
412 goto cleanup;
413 }
414 }
415
416 /*
417 * Validate and setup table sizes
418 */
419 if (tf_em_validate_num_entries(tbl_scope_cb, parms))
420 goto cleanup;
421
422 for (dir = 0; dir < TF_DIR_MAX; dir++) {
423 /*
424 * Allocate tables and signal configuration to FW
425 */
426 rc = tf_em_ctx_reg(tfp, tbl_scope_cb, dir);
427 if (rc) {
428 TFP_DRV_LOG(ERR,
429 "EEM: Unable to register for EEM ctx,"
430 " rc:%s\n",
431 strerror(-rc));
432 goto cleanup;
433 }
434
435 em_tables = tbl_scope_cb->em_ctx_info[dir].em_tables;
436 rc = tf_msg_em_cfg(tfp,
437 em_tables[TF_KEY0_TABLE].num_entries,
438 em_tables[TF_KEY0_TABLE].ctx_id,
439 em_tables[TF_KEY1_TABLE].ctx_id,
440 em_tables[TF_RECORD_TABLE].ctx_id,
441 em_tables[TF_EFC_TABLE].ctx_id,
442 parms->hw_flow_cache_flush_timer,
443 dir);
444 if (rc) {
445 TFP_DRV_LOG(ERR,
446 "TBL: Unable to configure EEM in firmware"
447 " rc:%s\n",
448 strerror(-rc));
449 goto cleanup_full;
450 }
451
452 rc = tf_msg_em_op(tfp,
453 dir,
454 HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_ENABLE);
455
456 if (rc) {
457 TFP_DRV_LOG(ERR,
458 "EEM: Unable to enable EEM in firmware"
459 " rc:%s\n",
460 strerror(-rc));
461 goto cleanup_full;
462 }
463
464 /* Allocate the pool of offsets of the external memory.
465 * Initially, this is a single fixed size pool for all external
466 * actions related to a single table scope.
467 */
468 rc = tf_create_tbl_pool_external(dir,
469 tbl_scope_cb,
470 em_tables[TF_RECORD_TABLE].num_entries,
471 em_tables[TF_RECORD_TABLE].entry_size);
472 if (rc) {
473 TFP_DRV_LOG(ERR,
474 "%s TBL: Unable to allocate idx pools %s\n",
475 tf_dir_2_str(dir),
476 strerror(-rc));
477 goto cleanup_full;
478 }
479 }
480
481 return 0;
482
483 cleanup_full:
484 free_parms.tbl_scope_id = parms->tbl_scope_id;
485 tf_em_ext_free(tfp, &free_parms);
486 return -EINVAL;
487
488 cleanup:
489 /* Free Table control block */
490 fparms.rm_db = eem_db[TF_DIR_RX];
491 fparms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
492 fparms.index = parms->tbl_scope_id;
493 tf_rm_free(&fparms);
494 return -EINVAL;
495 }
496
497 int
tf_em_ext_free(struct tf * tfp,struct tf_free_tbl_scope_parms * parms)498 tf_em_ext_free(struct tf *tfp,
499 struct tf_free_tbl_scope_parms *parms)
500 {
501 int rc = 0;
502 enum tf_dir dir;
503 struct tf_tbl_scope_cb *tbl_scope_cb;
504 struct tf_rm_free_parms aparms = { 0 };
505
506 tbl_scope_cb = tbl_scope_cb_find(parms->tbl_scope_id);
507
508 if (tbl_scope_cb == NULL) {
509 TFP_DRV_LOG(ERR, "Table scope error\n");
510 return -EINVAL;
511 }
512
513 /* Free Table control block */
514 aparms.rm_db = eem_db[TF_DIR_RX];
515 aparms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
516 aparms.index = parms->tbl_scope_id;
517 rc = tf_rm_free(&aparms);
518 if (rc) {
519 TFP_DRV_LOG(ERR,
520 "Failed to free table scope\n");
521 }
522
523 /* free table scope locks */
524 for (dir = 0; dir < TF_DIR_MAX; dir++) {
525 /* Free associated external pools
526 */
527 tf_destroy_tbl_pool_external(dir,
528 tbl_scope_cb);
529 tf_msg_em_op(tfp,
530 dir,
531 HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_DISABLE);
532
533 /* free table scope and all associated resources */
534 tf_em_ctx_unreg(tfp, tbl_scope_cb, dir);
535 }
536
537 tbl_scopes[parms->tbl_scope_id].tbl_scope_id = TF_TBL_SCOPE_INVALID;
538 return rc;
539 }
540