1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation
3 */
4
5 #include <string.h>
6 #include <stdio.h>
7
8 #include <rte_common.h>
9 #include <rte_mbuf.h>
10 #include <rte_malloc.h>
11 #include <rte_string_fns.h>
12
13 #include "rte_pipeline.h"
14
15 #define RTE_TABLE_INVALID UINT32_MAX
16
17 #ifdef RTE_PIPELINE_STATS_COLLECT
18
19 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \
20 ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); })
21
22 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \
23 ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; })
24
25 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) \
26 ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; })
27
28 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \
29 ({ \
30 uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \
31 mask ^= (p)->pkts_drop_mask; \
32 (counter) += __builtin_popcountll(mask); \
33 })
34
35 #else
36
37 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)
38 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)
39 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)
40 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)
41
42 #endif
43
44 struct rte_port_in {
45 /* Input parameters */
46 struct rte_port_in_ops ops;
47 rte_pipeline_port_in_action_handler f_action;
48 void *arg_ah;
49 uint32_t burst_size;
50
51 /* The table to which this port is connected */
52 uint32_t table_id;
53
54 /* Handle to low-level port */
55 void *h_port;
56
57 /* List of enabled ports */
58 struct rte_port_in *next;
59
60 /* Statistics */
61 uint64_t n_pkts_dropped_by_ah;
62 };
63
64 struct rte_port_out {
65 /* Input parameters */
66 struct rte_port_out_ops ops;
67 rte_pipeline_port_out_action_handler f_action;
68 void *arg_ah;
69
70 /* Handle to low-level port */
71 void *h_port;
72
73 /* Statistics */
74 uint64_t n_pkts_dropped_by_ah;
75 };
76
77 struct rte_table {
78 /* Input parameters */
79 struct rte_table_ops ops;
80 rte_pipeline_table_action_handler_hit f_action_hit;
81 rte_pipeline_table_action_handler_miss f_action_miss;
82 void *arg_ah;
83 struct rte_pipeline_table_entry *default_entry;
84 uint32_t entry_size;
85
86 uint32_t table_next_id;
87 uint32_t table_next_id_valid;
88
89 /* Handle to the low-level table object */
90 void *h_table;
91
92 /* Statistics */
93 uint64_t n_pkts_dropped_by_lkp_hit_ah;
94 uint64_t n_pkts_dropped_by_lkp_miss_ah;
95 uint64_t n_pkts_dropped_lkp_hit;
96 uint64_t n_pkts_dropped_lkp_miss;
97 };
98
99 #define RTE_PIPELINE_MAX_NAME_SZ 124
100
101 struct rte_pipeline {
102 /* Input parameters */
103 char name[RTE_PIPELINE_MAX_NAME_SZ];
104 int socket_id;
105 uint32_t offset_port_id;
106
107 /* Internal tables */
108 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
109 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
110 struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
111
112 /* Occupancy of internal tables */
113 uint32_t num_ports_in;
114 uint32_t num_ports_out;
115 uint32_t num_tables;
116
117 /* List of enabled ports */
118 uint64_t enabled_port_in_mask;
119 struct rte_port_in *port_in_next;
120
121 /* Pipeline run structures */
122 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
123 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
124 uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
125 uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
126 uint64_t pkts_mask;
127 uint64_t n_pkts_ah_drop;
128 uint64_t pkts_drop_mask;
129 } __rte_cache_aligned;
130
131 static inline uint32_t
rte_mask_get_next(uint64_t mask,uint32_t pos)132 rte_mask_get_next(uint64_t mask, uint32_t pos)
133 {
134 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
135 (mask >> ((pos + 1) & 0x3F));
136 return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
137 }
138
139 static inline uint32_t
rte_mask_get_prev(uint64_t mask,uint32_t pos)140 rte_mask_get_prev(uint64_t mask, uint32_t pos)
141 {
142 uint64_t mask_rot = (mask >> (pos & 0x3F)) |
143 (mask << ((64 - pos) & 0x3F));
144 return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
145 }
146
147 static void
148 rte_pipeline_table_free(struct rte_table *table);
149
150 static void
151 rte_pipeline_port_in_free(struct rte_port_in *port);
152
153 static void
154 rte_pipeline_port_out_free(struct rte_port_out *port);
155
156 /*
157 * Pipeline
158 *
159 */
160 static int
rte_pipeline_check_params(struct rte_pipeline_params * params)161 rte_pipeline_check_params(struct rte_pipeline_params *params)
162 {
163 if (params == NULL) {
164 RTE_LOG(ERR, PIPELINE,
165 "%s: Incorrect value for parameter params\n", __func__);
166 return -EINVAL;
167 }
168
169 /* name */
170 if (params->name == NULL) {
171 RTE_LOG(ERR, PIPELINE,
172 "%s: Incorrect value for parameter name\n", __func__);
173 return -EINVAL;
174 }
175
176 /* socket */
177 if (params->socket_id < 0) {
178 RTE_LOG(ERR, PIPELINE,
179 "%s: Incorrect value for parameter socket_id\n",
180 __func__);
181 return -EINVAL;
182 }
183
184 return 0;
185 }
186
187 struct rte_pipeline *
rte_pipeline_create(struct rte_pipeline_params * params)188 rte_pipeline_create(struct rte_pipeline_params *params)
189 {
190 struct rte_pipeline *p;
191 int status;
192
193 /* Check input parameters */
194 status = rte_pipeline_check_params(params);
195 if (status != 0) {
196 RTE_LOG(ERR, PIPELINE,
197 "%s: Pipeline params check failed (%d)\n",
198 __func__, status);
199 return NULL;
200 }
201
202 /* Allocate memory for the pipeline on requested socket */
203 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
204 RTE_CACHE_LINE_SIZE, params->socket_id);
205
206 if (p == NULL) {
207 RTE_LOG(ERR, PIPELINE,
208 "%s: Pipeline memory allocation failed\n", __func__);
209 return NULL;
210 }
211
212 /* Save input parameters */
213 strlcpy(p->name, params->name, RTE_PIPELINE_MAX_NAME_SZ);
214 p->socket_id = params->socket_id;
215 p->offset_port_id = params->offset_port_id;
216
217 /* Initialize pipeline internal data structure */
218 p->num_ports_in = 0;
219 p->num_ports_out = 0;
220 p->num_tables = 0;
221 p->enabled_port_in_mask = 0;
222 p->port_in_next = NULL;
223 p->pkts_mask = 0;
224 p->n_pkts_ah_drop = 0;
225
226 return p;
227 }
228
229 int
rte_pipeline_free(struct rte_pipeline * p)230 rte_pipeline_free(struct rte_pipeline *p)
231 {
232 uint32_t i;
233
234 /* Check input parameters */
235 if (p == NULL) {
236 RTE_LOG(ERR, PIPELINE,
237 "%s: rte_pipeline parameter is NULL\n", __func__);
238 return -EINVAL;
239 }
240
241 /* Free input ports */
242 for (i = 0; i < p->num_ports_in; i++) {
243 struct rte_port_in *port = &p->ports_in[i];
244
245 rte_pipeline_port_in_free(port);
246 }
247
248 /* Free tables */
249 for (i = 0; i < p->num_tables; i++) {
250 struct rte_table *table = &p->tables[i];
251
252 rte_pipeline_table_free(table);
253 }
254
255 /* Free output ports */
256 for (i = 0; i < p->num_ports_out; i++) {
257 struct rte_port_out *port = &p->ports_out[i];
258
259 rte_pipeline_port_out_free(port);
260 }
261
262 /* Free pipeline memory */
263 rte_free(p);
264
265 return 0;
266 }
267
268 /*
269 * Table
270 *
271 */
272 static int
rte_table_check_params(struct rte_pipeline * p,struct rte_pipeline_table_params * params,uint32_t * table_id)273 rte_table_check_params(struct rte_pipeline *p,
274 struct rte_pipeline_table_params *params,
275 uint32_t *table_id)
276 {
277 if (p == NULL) {
278 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
279 __func__);
280 return -EINVAL;
281 }
282 if (params == NULL) {
283 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
284 __func__);
285 return -EINVAL;
286 }
287 if (table_id == NULL) {
288 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
289 __func__);
290 return -EINVAL;
291 }
292
293 /* ops */
294 if (params->ops == NULL) {
295 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
296 __func__);
297 return -EINVAL;
298 }
299
300 if (params->ops->f_create == NULL) {
301 RTE_LOG(ERR, PIPELINE,
302 "%s: f_create function pointer is NULL\n", __func__);
303 return -EINVAL;
304 }
305
306 if (params->ops->f_lookup == NULL) {
307 RTE_LOG(ERR, PIPELINE,
308 "%s: f_lookup function pointer is NULL\n", __func__);
309 return -EINVAL;
310 }
311
312 /* De we have room for one more table? */
313 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
314 RTE_LOG(ERR, PIPELINE,
315 "%s: Incorrect value for num_tables parameter\n",
316 __func__);
317 return -EINVAL;
318 }
319
320 return 0;
321 }
322
323 int
rte_pipeline_table_create(struct rte_pipeline * p,struct rte_pipeline_table_params * params,uint32_t * table_id)324 rte_pipeline_table_create(struct rte_pipeline *p,
325 struct rte_pipeline_table_params *params,
326 uint32_t *table_id)
327 {
328 struct rte_table *table;
329 struct rte_pipeline_table_entry *default_entry;
330 void *h_table;
331 uint32_t entry_size, id;
332 int status;
333
334 /* Check input arguments */
335 status = rte_table_check_params(p, params, table_id);
336 if (status != 0)
337 return status;
338
339 id = p->num_tables;
340 table = &p->tables[id];
341
342 /* Allocate space for the default table entry */
343 entry_size = sizeof(struct rte_pipeline_table_entry) +
344 params->action_data_size;
345 default_entry = rte_zmalloc_socket(
346 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
347 if (default_entry == NULL) {
348 RTE_LOG(ERR, PIPELINE,
349 "%s: Failed to allocate default entry\n", __func__);
350 return -EINVAL;
351 }
352
353 /* Create the table */
354 h_table = params->ops->f_create(params->arg_create, p->socket_id,
355 entry_size);
356 if (h_table == NULL) {
357 rte_free(default_entry);
358 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
359 return -EINVAL;
360 }
361
362 /* Commit current table to the pipeline */
363 p->num_tables++;
364 *table_id = id;
365
366 /* Save input parameters */
367 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
368 table->f_action_hit = params->f_action_hit;
369 table->f_action_miss = params->f_action_miss;
370 table->arg_ah = params->arg_ah;
371 table->entry_size = entry_size;
372
373 /* Clear the lookup miss actions (to be set later through API) */
374 table->default_entry = default_entry;
375 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
376
377 /* Initialize table internal data structure */
378 table->h_table = h_table;
379 table->table_next_id = 0;
380 table->table_next_id_valid = 0;
381
382 return 0;
383 }
384
385 void
rte_pipeline_table_free(struct rte_table * table)386 rte_pipeline_table_free(struct rte_table *table)
387 {
388 if (table->ops.f_free != NULL)
389 table->ops.f_free(table->h_table);
390
391 rte_free(table->default_entry);
392 }
393
394 int
rte_pipeline_table_default_entry_add(struct rte_pipeline * p,uint32_t table_id,struct rte_pipeline_table_entry * default_entry,struct rte_pipeline_table_entry ** default_entry_ptr)395 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
396 uint32_t table_id,
397 struct rte_pipeline_table_entry *default_entry,
398 struct rte_pipeline_table_entry **default_entry_ptr)
399 {
400 struct rte_table *table;
401
402 /* Check input arguments */
403 if (p == NULL) {
404 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
405 __func__);
406 return -EINVAL;
407 }
408
409 if (default_entry == NULL) {
410 RTE_LOG(ERR, PIPELINE,
411 "%s: default_entry parameter is NULL\n", __func__);
412 return -EINVAL;
413 }
414
415 if (table_id >= p->num_tables) {
416 RTE_LOG(ERR, PIPELINE,
417 "%s: table_id %d out of range\n", __func__, table_id);
418 return -EINVAL;
419 }
420
421 table = &p->tables[table_id];
422
423 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
424 table->table_next_id_valid &&
425 (default_entry->table_id != table->table_next_id)) {
426 RTE_LOG(ERR, PIPELINE,
427 "%s: Tree-like topologies not allowed\n", __func__);
428 return -EINVAL;
429 }
430
431 /* Set the lookup miss actions */
432 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
433 (table->table_next_id_valid == 0)) {
434 table->table_next_id = default_entry->table_id;
435 table->table_next_id_valid = 1;
436 }
437
438 memcpy(table->default_entry, default_entry, table->entry_size);
439
440 *default_entry_ptr = table->default_entry;
441 return 0;
442 }
443
444 int
rte_pipeline_table_default_entry_delete(struct rte_pipeline * p,uint32_t table_id,struct rte_pipeline_table_entry * entry)445 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
446 uint32_t table_id,
447 struct rte_pipeline_table_entry *entry)
448 {
449 struct rte_table *table;
450
451 /* Check input arguments */
452 if (p == NULL) {
453 RTE_LOG(ERR, PIPELINE,
454 "%s: pipeline parameter is NULL\n", __func__);
455 return -EINVAL;
456 }
457
458 if (table_id >= p->num_tables) {
459 RTE_LOG(ERR, PIPELINE,
460 "%s: table_id %d out of range\n", __func__, table_id);
461 return -EINVAL;
462 }
463
464 table = &p->tables[table_id];
465
466 /* Save the current contents of the default entry */
467 if (entry)
468 memcpy(entry, table->default_entry, table->entry_size);
469
470 /* Clear the lookup miss actions */
471 memset(table->default_entry, 0, table->entry_size);
472 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
473
474 return 0;
475 }
476
477 int
rte_pipeline_table_entry_add(struct rte_pipeline * p,uint32_t table_id,void * key,struct rte_pipeline_table_entry * entry,int * key_found,struct rte_pipeline_table_entry ** entry_ptr)478 rte_pipeline_table_entry_add(struct rte_pipeline *p,
479 uint32_t table_id,
480 void *key,
481 struct rte_pipeline_table_entry *entry,
482 int *key_found,
483 struct rte_pipeline_table_entry **entry_ptr)
484 {
485 struct rte_table *table;
486
487 /* Check input arguments */
488 if (p == NULL) {
489 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
490 __func__);
491 return -EINVAL;
492 }
493
494 if (key == NULL) {
495 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
496 return -EINVAL;
497 }
498
499 if (entry == NULL) {
500 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
501 __func__);
502 return -EINVAL;
503 }
504
505 if (table_id >= p->num_tables) {
506 RTE_LOG(ERR, PIPELINE,
507 "%s: table_id %d out of range\n", __func__, table_id);
508 return -EINVAL;
509 }
510
511 table = &p->tables[table_id];
512
513 if (table->ops.f_add == NULL) {
514 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
515 __func__);
516 return -EINVAL;
517 }
518
519 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
520 table->table_next_id_valid &&
521 (entry->table_id != table->table_next_id)) {
522 RTE_LOG(ERR, PIPELINE,
523 "%s: Tree-like topologies not allowed\n", __func__);
524 return -EINVAL;
525 }
526
527 /* Add entry */
528 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
529 (table->table_next_id_valid == 0)) {
530 table->table_next_id = entry->table_id;
531 table->table_next_id_valid = 1;
532 }
533
534 return (table->ops.f_add)(table->h_table, key, (void *) entry,
535 key_found, (void **) entry_ptr);
536 }
537
538 int
rte_pipeline_table_entry_delete(struct rte_pipeline * p,uint32_t table_id,void * key,int * key_found,struct rte_pipeline_table_entry * entry)539 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
540 uint32_t table_id,
541 void *key,
542 int *key_found,
543 struct rte_pipeline_table_entry *entry)
544 {
545 struct rte_table *table;
546
547 /* Check input arguments */
548 if (p == NULL) {
549 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
550 __func__);
551 return -EINVAL;
552 }
553
554 if (key == NULL) {
555 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
556 __func__);
557 return -EINVAL;
558 }
559
560 if (table_id >= p->num_tables) {
561 RTE_LOG(ERR, PIPELINE,
562 "%s: table_id %d out of range\n", __func__, table_id);
563 return -EINVAL;
564 }
565
566 table = &p->tables[table_id];
567
568 if (table->ops.f_delete == NULL) {
569 RTE_LOG(ERR, PIPELINE,
570 "%s: f_delete function pointer NULL\n", __func__);
571 return -EINVAL;
572 }
573
574 return (table->ops.f_delete)(table->h_table, key, key_found, entry);
575 }
576
rte_pipeline_table_entry_add_bulk(struct rte_pipeline * p,uint32_t table_id,void ** keys,struct rte_pipeline_table_entry ** entries,uint32_t n_keys,int * key_found,struct rte_pipeline_table_entry ** entries_ptr)577 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
578 uint32_t table_id,
579 void **keys,
580 struct rte_pipeline_table_entry **entries,
581 uint32_t n_keys,
582 int *key_found,
583 struct rte_pipeline_table_entry **entries_ptr)
584 {
585 struct rte_table *table;
586 uint32_t i;
587
588 /* Check input arguments */
589 if (p == NULL) {
590 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
591 __func__);
592 return -EINVAL;
593 }
594
595 if (keys == NULL) {
596 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
597 return -EINVAL;
598 }
599
600 if (entries == NULL) {
601 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
602 __func__);
603 return -EINVAL;
604 }
605
606 if (table_id >= p->num_tables) {
607 RTE_LOG(ERR, PIPELINE,
608 "%s: table_id %d out of range\n", __func__, table_id);
609 return -EINVAL;
610 }
611
612 table = &p->tables[table_id];
613
614 if (table->ops.f_add_bulk == NULL) {
615 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
616 __func__);
617 return -EINVAL;
618 }
619
620 for (i = 0; i < n_keys; i++) {
621 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
622 table->table_next_id_valid &&
623 (entries[i]->table_id != table->table_next_id)) {
624 RTE_LOG(ERR, PIPELINE,
625 "%s: Tree-like topologies not allowed\n", __func__);
626 return -EINVAL;
627 }
628 }
629
630 /* Add entry */
631 for (i = 0; i < n_keys; i++) {
632 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
633 (table->table_next_id_valid == 0)) {
634 table->table_next_id = entries[i]->table_id;
635 table->table_next_id_valid = 1;
636 }
637 }
638
639 return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
640 n_keys, key_found, (void **) entries_ptr);
641 }
642
rte_pipeline_table_entry_delete_bulk(struct rte_pipeline * p,uint32_t table_id,void ** keys,uint32_t n_keys,int * key_found,struct rte_pipeline_table_entry ** entries)643 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
644 uint32_t table_id,
645 void **keys,
646 uint32_t n_keys,
647 int *key_found,
648 struct rte_pipeline_table_entry **entries)
649 {
650 struct rte_table *table;
651
652 /* Check input arguments */
653 if (p == NULL) {
654 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
655 __func__);
656 return -EINVAL;
657 }
658
659 if (keys == NULL) {
660 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
661 __func__);
662 return -EINVAL;
663 }
664
665 if (table_id >= p->num_tables) {
666 RTE_LOG(ERR, PIPELINE,
667 "%s: table_id %d out of range\n", __func__, table_id);
668 return -EINVAL;
669 }
670
671 table = &p->tables[table_id];
672
673 if (table->ops.f_delete_bulk == NULL) {
674 RTE_LOG(ERR, PIPELINE,
675 "%s: f_delete function pointer NULL\n", __func__);
676 return -EINVAL;
677 }
678
679 return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
680 (void **) entries);
681 }
682
683 /*
684 * Port
685 *
686 */
687 static int
rte_pipeline_port_in_check_params(struct rte_pipeline * p,struct rte_pipeline_port_in_params * params,uint32_t * port_id)688 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
689 struct rte_pipeline_port_in_params *params,
690 uint32_t *port_id)
691 {
692 if (p == NULL) {
693 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
694 __func__);
695 return -EINVAL;
696 }
697 if (params == NULL) {
698 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
699 return -EINVAL;
700 }
701 if (port_id == NULL) {
702 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
703 __func__);
704 return -EINVAL;
705 }
706
707 /* ops */
708 if (params->ops == NULL) {
709 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
710 __func__);
711 return -EINVAL;
712 }
713
714 if (params->ops->f_create == NULL) {
715 RTE_LOG(ERR, PIPELINE,
716 "%s: f_create function pointer NULL\n", __func__);
717 return -EINVAL;
718 }
719
720 if (params->ops->f_rx == NULL) {
721 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
722 __func__);
723 return -EINVAL;
724 }
725
726 /* burst_size */
727 if ((params->burst_size == 0) ||
728 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
729 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
730 __func__);
731 return -EINVAL;
732 }
733
734 /* Do we have room for one more port? */
735 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
736 RTE_LOG(ERR, PIPELINE,
737 "%s: invalid value for num_ports_in\n", __func__);
738 return -EINVAL;
739 }
740
741 return 0;
742 }
743
744 static int
rte_pipeline_port_out_check_params(struct rte_pipeline * p,struct rte_pipeline_port_out_params * params,uint32_t * port_id)745 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
746 struct rte_pipeline_port_out_params *params,
747 uint32_t *port_id)
748 {
749 if (p == NULL) {
750 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
751 __func__);
752 return -EINVAL;
753 }
754
755 if (params == NULL) {
756 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
757 return -EINVAL;
758 }
759
760 if (port_id == NULL) {
761 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
762 __func__);
763 return -EINVAL;
764 }
765
766 /* ops */
767 if (params->ops == NULL) {
768 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
769 __func__);
770 return -EINVAL;
771 }
772
773 if (params->ops->f_create == NULL) {
774 RTE_LOG(ERR, PIPELINE,
775 "%s: f_create function pointer NULL\n", __func__);
776 return -EINVAL;
777 }
778
779 if (params->ops->f_tx == NULL) {
780 RTE_LOG(ERR, PIPELINE,
781 "%s: f_tx function pointer NULL\n", __func__);
782 return -EINVAL;
783 }
784
785 if (params->ops->f_tx_bulk == NULL) {
786 RTE_LOG(ERR, PIPELINE,
787 "%s: f_tx_bulk function pointer NULL\n", __func__);
788 return -EINVAL;
789 }
790
791 /* Do we have room for one more port? */
792 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
793 RTE_LOG(ERR, PIPELINE,
794 "%s: invalid value for num_ports_out\n", __func__);
795 return -EINVAL;
796 }
797
798 return 0;
799 }
800
801 int
rte_pipeline_port_in_create(struct rte_pipeline * p,struct rte_pipeline_port_in_params * params,uint32_t * port_id)802 rte_pipeline_port_in_create(struct rte_pipeline *p,
803 struct rte_pipeline_port_in_params *params,
804 uint32_t *port_id)
805 {
806 struct rte_port_in *port;
807 void *h_port;
808 uint32_t id;
809 int status;
810
811 /* Check input arguments */
812 status = rte_pipeline_port_in_check_params(p, params, port_id);
813 if (status != 0)
814 return status;
815
816 id = p->num_ports_in;
817 port = &p->ports_in[id];
818
819 /* Create the port */
820 h_port = params->ops->f_create(params->arg_create, p->socket_id);
821 if (h_port == NULL) {
822 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
823 return -EINVAL;
824 }
825
826 /* Commit current table to the pipeline */
827 p->num_ports_in++;
828 *port_id = id;
829
830 /* Save input parameters */
831 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
832 port->f_action = params->f_action;
833 port->arg_ah = params->arg_ah;
834 port->burst_size = params->burst_size;
835
836 /* Initialize port internal data structure */
837 port->table_id = RTE_TABLE_INVALID;
838 port->h_port = h_port;
839 port->next = NULL;
840
841 return 0;
842 }
843
844 void
rte_pipeline_port_in_free(struct rte_port_in * port)845 rte_pipeline_port_in_free(struct rte_port_in *port)
846 {
847 if (port->ops.f_free != NULL)
848 port->ops.f_free(port->h_port);
849 }
850
851 int
rte_pipeline_port_out_create(struct rte_pipeline * p,struct rte_pipeline_port_out_params * params,uint32_t * port_id)852 rte_pipeline_port_out_create(struct rte_pipeline *p,
853 struct rte_pipeline_port_out_params *params,
854 uint32_t *port_id)
855 {
856 struct rte_port_out *port;
857 void *h_port;
858 uint32_t id;
859 int status;
860
861 /* Check input arguments */
862 status = rte_pipeline_port_out_check_params(p, params, port_id);
863 if (status != 0)
864 return status;
865
866 id = p->num_ports_out;
867 port = &p->ports_out[id];
868
869 /* Create the port */
870 h_port = params->ops->f_create(params->arg_create, p->socket_id);
871 if (h_port == NULL) {
872 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
873 return -EINVAL;
874 }
875
876 /* Commit current table to the pipeline */
877 p->num_ports_out++;
878 *port_id = id;
879
880 /* Save input parameters */
881 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
882 port->f_action = params->f_action;
883 port->arg_ah = params->arg_ah;
884
885 /* Initialize port internal data structure */
886 port->h_port = h_port;
887
888 return 0;
889 }
890
891 void
rte_pipeline_port_out_free(struct rte_port_out * port)892 rte_pipeline_port_out_free(struct rte_port_out *port)
893 {
894 if (port->ops.f_free != NULL)
895 port->ops.f_free(port->h_port);
896 }
897
898 int
rte_pipeline_port_in_connect_to_table(struct rte_pipeline * p,uint32_t port_id,uint32_t table_id)899 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
900 uint32_t port_id,
901 uint32_t table_id)
902 {
903 struct rte_port_in *port;
904
905 /* Check input arguments */
906 if (p == NULL) {
907 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
908 __func__);
909 return -EINVAL;
910 }
911
912 if (port_id >= p->num_ports_in) {
913 RTE_LOG(ERR, PIPELINE,
914 "%s: port IN ID %u is out of range\n",
915 __func__, port_id);
916 return -EINVAL;
917 }
918
919 if (table_id >= p->num_tables) {
920 RTE_LOG(ERR, PIPELINE,
921 "%s: Table ID %u is out of range\n",
922 __func__, table_id);
923 return -EINVAL;
924 }
925
926 port = &p->ports_in[port_id];
927 port->table_id = table_id;
928
929 return 0;
930 }
931
932 int
rte_pipeline_port_in_enable(struct rte_pipeline * p,uint32_t port_id)933 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
934 {
935 struct rte_port_in *port, *port_prev, *port_next;
936 uint64_t port_mask;
937 uint32_t port_prev_id, port_next_id;
938
939 /* Check input arguments */
940 if (p == NULL) {
941 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
942 __func__);
943 return -EINVAL;
944 }
945
946 if (port_id >= p->num_ports_in) {
947 RTE_LOG(ERR, PIPELINE,
948 "%s: port IN ID %u is out of range\n",
949 __func__, port_id);
950 return -EINVAL;
951 }
952
953 port = &p->ports_in[port_id];
954
955 /* Return if current input port is already enabled */
956 port_mask = 1LLU << port_id;
957 if (p->enabled_port_in_mask & port_mask)
958 return 0;
959
960 p->enabled_port_in_mask |= port_mask;
961
962 /* Add current input port to the pipeline chain of enabled ports */
963 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
964 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
965
966 port_prev = &p->ports_in[port_prev_id];
967 port_next = &p->ports_in[port_next_id];
968
969 port_prev->next = port;
970 port->next = port_next;
971
972 /* Check if list of enabled ports was previously empty */
973 if (p->enabled_port_in_mask == port_mask)
974 p->port_in_next = port;
975
976 return 0;
977 }
978
979 int
rte_pipeline_port_in_disable(struct rte_pipeline * p,uint32_t port_id)980 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
981 {
982 struct rte_port_in *port, *port_prev, *port_next;
983 uint64_t port_mask;
984 uint32_t port_prev_id, port_next_id;
985
986 /* Check input arguments */
987 if (p == NULL) {
988 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
989 __func__);
990 return -EINVAL;
991 }
992
993 if (port_id >= p->num_ports_in) {
994 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
995 __func__, port_id);
996 return -EINVAL;
997 }
998
999 port = &p->ports_in[port_id];
1000
1001 /* Return if current input port is already disabled */
1002 port_mask = 1LLU << port_id;
1003 if ((p->enabled_port_in_mask & port_mask) == 0)
1004 return 0;
1005
1006 p->enabled_port_in_mask &= ~port_mask;
1007
1008 /* Return if no other enabled ports */
1009 if (p->enabled_port_in_mask == 0) {
1010 p->port_in_next = NULL;
1011
1012 return 0;
1013 }
1014
1015 /* Add current input port to the pipeline chain of enabled ports */
1016 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1017 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1018
1019 port_prev = &p->ports_in[port_prev_id];
1020 port_next = &p->ports_in[port_next_id];
1021
1022 port_prev->next = port_next;
1023
1024 /* Check if the port which has just been disabled is next to serve */
1025 if (port == p->port_in_next)
1026 p->port_in_next = port_next;
1027
1028 return 0;
1029 }
1030
1031 /*
1032 * Pipeline run-time
1033 *
1034 */
1035 int
rte_pipeline_check(struct rte_pipeline * p)1036 rte_pipeline_check(struct rte_pipeline *p)
1037 {
1038 uint32_t port_in_id;
1039
1040 /* Check input arguments */
1041 if (p == NULL) {
1042 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1043 __func__);
1044 return -EINVAL;
1045 }
1046
1047 /* Check that pipeline has at least one input port, one table and one
1048 output port */
1049 if (p->num_ports_in == 0) {
1050 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1051 __func__);
1052 return -EINVAL;
1053 }
1054 if (p->num_tables == 0) {
1055 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1056 __func__);
1057 return -EINVAL;
1058 }
1059 if (p->num_ports_out == 0) {
1060 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1061 __func__);
1062 return -EINVAL;
1063 }
1064
1065 /* Check that all input ports are connected */
1066 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1067 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1068
1069 if (port_in->table_id == RTE_TABLE_INVALID) {
1070 RTE_LOG(ERR, PIPELINE,
1071 "%s: Port IN ID %u is not connected\n",
1072 __func__, port_in_id);
1073 return -EINVAL;
1074 }
1075 }
1076
1077 return 0;
1078 }
1079
1080 static inline void
rte_pipeline_compute_masks(struct rte_pipeline * p,uint64_t pkts_mask)1081 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1082 {
1083 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1084 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1085 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1086 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1087
1088 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1089 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1090 uint32_t i;
1091
1092 for (i = 0; i < n_pkts; i++) {
1093 uint64_t pkt_mask = 1LLU << i;
1094 uint32_t pos = p->entries[i]->action;
1095
1096 p->action_mask1[pos] |= pkt_mask;
1097 }
1098 } else {
1099 uint32_t i;
1100
1101 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1102 uint64_t pkt_mask = 1LLU << i;
1103 uint32_t pos;
1104
1105 if ((pkt_mask & pkts_mask) == 0)
1106 continue;
1107
1108 pos = p->entries[i]->action;
1109 p->action_mask1[pos] |= pkt_mask;
1110 }
1111 }
1112 }
1113
1114 static inline void
rte_pipeline_action_handler_port_bulk(struct rte_pipeline * p,uint64_t pkts_mask,uint32_t port_id)1115 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1116 uint64_t pkts_mask, uint32_t port_id)
1117 {
1118 struct rte_port_out *port_out = &p->ports_out[port_id];
1119
1120 p->pkts_mask = pkts_mask;
1121
1122 /* Output port user actions */
1123 if (port_out->f_action != NULL) {
1124 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah);
1125
1126 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1127 port_out->n_pkts_dropped_by_ah);
1128 }
1129
1130 /* Output port TX */
1131 if (p->pkts_mask != 0)
1132 port_out->ops.f_tx_bulk(port_out->h_port,
1133 p->pkts,
1134 p->pkts_mask);
1135 }
1136
1137 static inline void
rte_pipeline_action_handler_port(struct rte_pipeline * p,uint64_t pkts_mask)1138 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1139 {
1140 p->pkts_mask = pkts_mask;
1141
1142 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1143 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1144 uint32_t i;
1145
1146 for (i = 0; i < n_pkts; i++) {
1147 struct rte_mbuf *pkt = p->pkts[i];
1148 uint32_t port_out_id = p->entries[i]->port_id;
1149 struct rte_port_out *port_out =
1150 &p->ports_out[port_out_id];
1151
1152 /* Output port user actions */
1153 if (port_out->f_action == NULL) /* Output port TX */
1154 port_out->ops.f_tx(port_out->h_port, pkt);
1155 else {
1156 uint64_t pkt_mask = 1LLU << i;
1157
1158 port_out->f_action(p,
1159 p->pkts,
1160 pkt_mask,
1161 port_out->arg_ah);
1162
1163 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1164 port_out->n_pkts_dropped_by_ah);
1165
1166 /* Output port TX */
1167 if (pkt_mask & p->pkts_mask)
1168 port_out->ops.f_tx(port_out->h_port,
1169 pkt);
1170 }
1171 }
1172 } else {
1173 uint32_t i;
1174
1175 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1176 uint64_t pkt_mask = 1LLU << i;
1177 struct rte_mbuf *pkt;
1178 struct rte_port_out *port_out;
1179 uint32_t port_out_id;
1180
1181 if ((pkt_mask & pkts_mask) == 0)
1182 continue;
1183
1184 pkt = p->pkts[i];
1185 port_out_id = p->entries[i]->port_id;
1186 port_out = &p->ports_out[port_out_id];
1187
1188 /* Output port user actions */
1189 if (port_out->f_action == NULL) /* Output port TX */
1190 port_out->ops.f_tx(port_out->h_port, pkt);
1191 else {
1192 port_out->f_action(p,
1193 p->pkts,
1194 pkt_mask,
1195 port_out->arg_ah);
1196
1197 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1198 port_out->n_pkts_dropped_by_ah);
1199
1200 /* Output port TX */
1201 if (pkt_mask & p->pkts_mask)
1202 port_out->ops.f_tx(port_out->h_port,
1203 pkt);
1204 }
1205 }
1206 }
1207 }
1208
1209 static inline void
rte_pipeline_action_handler_port_meta(struct rte_pipeline * p,uint64_t pkts_mask)1210 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1211 uint64_t pkts_mask)
1212 {
1213 p->pkts_mask = pkts_mask;
1214
1215 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1216 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1217 uint32_t i;
1218
1219 for (i = 0; i < n_pkts; i++) {
1220 struct rte_mbuf *pkt = p->pkts[i];
1221 uint32_t port_out_id =
1222 RTE_MBUF_METADATA_UINT32(pkt,
1223 p->offset_port_id);
1224 struct rte_port_out *port_out = &p->ports_out[
1225 port_out_id];
1226
1227 /* Output port user actions */
1228 if (port_out->f_action == NULL) /* Output port TX */
1229 port_out->ops.f_tx(port_out->h_port, pkt);
1230 else {
1231 uint64_t pkt_mask = 1LLU << i;
1232
1233 port_out->f_action(p,
1234 p->pkts,
1235 pkt_mask,
1236 port_out->arg_ah);
1237
1238 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1239 port_out->n_pkts_dropped_by_ah);
1240
1241 /* Output port TX */
1242 if (pkt_mask & p->pkts_mask)
1243 port_out->ops.f_tx(port_out->h_port,
1244 pkt);
1245 }
1246 }
1247 } else {
1248 uint32_t i;
1249
1250 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1251 uint64_t pkt_mask = 1LLU << i;
1252 struct rte_mbuf *pkt;
1253 struct rte_port_out *port_out;
1254 uint32_t port_out_id;
1255
1256 if ((pkt_mask & pkts_mask) == 0)
1257 continue;
1258
1259 pkt = p->pkts[i];
1260 port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1261 p->offset_port_id);
1262 port_out = &p->ports_out[port_out_id];
1263
1264 /* Output port user actions */
1265 if (port_out->f_action == NULL) /* Output port TX */
1266 port_out->ops.f_tx(port_out->h_port, pkt);
1267 else {
1268 port_out->f_action(p,
1269 p->pkts,
1270 pkt_mask,
1271 port_out->arg_ah);
1272
1273 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1274 port_out->n_pkts_dropped_by_ah);
1275
1276 /* Output port TX */
1277 if (pkt_mask & p->pkts_mask)
1278 port_out->ops.f_tx(port_out->h_port,
1279 pkt);
1280 }
1281 }
1282 }
1283 }
1284
1285 static inline void
rte_pipeline_action_handler_drop(struct rte_pipeline * p,uint64_t pkts_mask)1286 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1287 {
1288 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1289 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1290 uint32_t i;
1291
1292 for (i = 0; i < n_pkts; i++)
1293 rte_pktmbuf_free(p->pkts[i]);
1294 } else {
1295 uint32_t i;
1296
1297 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1298 uint64_t pkt_mask = 1LLU << i;
1299
1300 if ((pkt_mask & pkts_mask) == 0)
1301 continue;
1302
1303 rte_pktmbuf_free(p->pkts[i]);
1304 }
1305 }
1306 }
1307
1308 int
rte_pipeline_run(struct rte_pipeline * p)1309 rte_pipeline_run(struct rte_pipeline *p)
1310 {
1311 struct rte_port_in *port_in = p->port_in_next;
1312 uint32_t n_pkts, table_id;
1313
1314 if (port_in == NULL)
1315 return 0;
1316
1317 /* Input port RX */
1318 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1319 port_in->burst_size);
1320 if (n_pkts == 0) {
1321 p->port_in_next = port_in->next;
1322 return 0;
1323 }
1324
1325 p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1326 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1327 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1328 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1329 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1330
1331 /* Input port user actions */
1332 if (port_in->f_action != NULL) {
1333 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah);
1334
1335 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1336 port_in->n_pkts_dropped_by_ah);
1337 }
1338
1339 /* Table */
1340 for (table_id = port_in->table_id; p->pkts_mask != 0; ) {
1341 struct rte_table *table;
1342 uint64_t lookup_hit_mask, lookup_miss_mask;
1343
1344 /* Lookup */
1345 table = &p->tables[table_id];
1346 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask,
1347 &lookup_hit_mask, (void **) p->entries);
1348 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask);
1349
1350 /* Lookup miss */
1351 if (lookup_miss_mask != 0) {
1352 struct rte_pipeline_table_entry *default_entry =
1353 table->default_entry;
1354
1355 p->pkts_mask = lookup_miss_mask;
1356
1357 /* Table user actions */
1358 if (table->f_action_miss != NULL) {
1359 table->f_action_miss(p,
1360 p->pkts,
1361 lookup_miss_mask,
1362 default_entry,
1363 table->arg_ah);
1364
1365 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1366 table->n_pkts_dropped_by_lkp_miss_ah);
1367 }
1368
1369 /* Table reserved actions */
1370 if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) &&
1371 (p->pkts_mask != 0))
1372 rte_pipeline_action_handler_port_bulk(p,
1373 p->pkts_mask,
1374 default_entry->port_id);
1375 else {
1376 uint32_t pos = default_entry->action;
1377
1378 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1379
1380 p->action_mask0[pos] |= p->pkts_mask;
1381
1382 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1383 table->n_pkts_dropped_lkp_miss);
1384 }
1385 }
1386
1387 /* Lookup hit */
1388 if (lookup_hit_mask != 0) {
1389 p->pkts_mask = lookup_hit_mask;
1390
1391 /* Table user actions */
1392 if (table->f_action_hit != NULL) {
1393 table->f_action_hit(p,
1394 p->pkts,
1395 lookup_hit_mask,
1396 p->entries,
1397 table->arg_ah);
1398
1399 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1400 table->n_pkts_dropped_by_lkp_hit_ah);
1401 }
1402
1403 /* Table reserved actions */
1404 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1405 rte_pipeline_compute_masks(p, p->pkts_mask);
1406 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1407 p->action_mask1[
1408 RTE_PIPELINE_ACTION_DROP];
1409 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1410 p->action_mask1[
1411 RTE_PIPELINE_ACTION_PORT];
1412 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1413 p->action_mask1[
1414 RTE_PIPELINE_ACTION_PORT_META];
1415 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1416 p->action_mask1[
1417 RTE_PIPELINE_ACTION_TABLE];
1418
1419 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1420 table->n_pkts_dropped_lkp_hit);
1421 }
1422
1423 /* Prepare for next iteration */
1424 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1425 table_id = table->table_next_id;
1426 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1427 }
1428
1429 /* Table reserved action PORT */
1430 rte_pipeline_action_handler_port(p,
1431 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1432
1433 /* Table reserved action PORT META */
1434 rte_pipeline_action_handler_port_meta(p,
1435 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1436
1437 /* Table reserved action DROP */
1438 rte_pipeline_action_handler_drop(p,
1439 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1440
1441 /* Pick candidate for next port IN to serve */
1442 p->port_in_next = port_in->next;
1443
1444 return (int) n_pkts;
1445 }
1446
1447 int
rte_pipeline_flush(struct rte_pipeline * p)1448 rte_pipeline_flush(struct rte_pipeline *p)
1449 {
1450 uint32_t port_id;
1451
1452 /* Check input arguments */
1453 if (p == NULL) {
1454 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1455 __func__);
1456 return -EINVAL;
1457 }
1458
1459 for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1460 struct rte_port_out *port = &p->ports_out[port_id];
1461
1462 if (port->ops.f_flush != NULL)
1463 port->ops.f_flush(port->h_port);
1464 }
1465
1466 return 0;
1467 }
1468
1469 int
rte_pipeline_port_out_packet_insert(struct rte_pipeline * p,uint32_t port_id,struct rte_mbuf * pkt)1470 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1471 uint32_t port_id, struct rte_mbuf *pkt)
1472 {
1473 struct rte_port_out *port_out = &p->ports_out[port_id];
1474
1475 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1476
1477 return 0;
1478 }
1479
rte_pipeline_ah_packet_hijack(struct rte_pipeline * p,uint64_t pkts_mask)1480 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p,
1481 uint64_t pkts_mask)
1482 {
1483 pkts_mask &= p->pkts_mask;
1484 p->pkts_mask &= ~pkts_mask;
1485
1486 return 0;
1487 }
1488
rte_pipeline_ah_packet_drop(struct rte_pipeline * p,uint64_t pkts_mask)1489 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p,
1490 uint64_t pkts_mask)
1491 {
1492 pkts_mask &= p->pkts_mask;
1493 p->pkts_mask &= ~pkts_mask;
1494 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask;
1495
1496 RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask);
1497 return 0;
1498 }
1499
rte_pipeline_port_in_stats_read(struct rte_pipeline * p,uint32_t port_id,struct rte_pipeline_port_in_stats * stats,int clear)1500 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1501 struct rte_pipeline_port_in_stats *stats, int clear)
1502 {
1503 struct rte_port_in *port;
1504 int retval;
1505
1506 if (p == NULL) {
1507 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1508 __func__);
1509 return -EINVAL;
1510 }
1511
1512 if (port_id >= p->num_ports_in) {
1513 RTE_LOG(ERR, PIPELINE,
1514 "%s: port IN ID %u is out of range\n",
1515 __func__, port_id);
1516 return -EINVAL;
1517 }
1518
1519 port = &p->ports_in[port_id];
1520
1521 if (port->ops.f_stats != NULL) {
1522 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1523 if (retval)
1524 return retval;
1525 } else if (stats != NULL)
1526 memset(&stats->stats, 0, sizeof(stats->stats));
1527
1528 if (stats != NULL)
1529 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1530
1531 if (clear != 0)
1532 port->n_pkts_dropped_by_ah = 0;
1533
1534 return 0;
1535 }
1536
rte_pipeline_port_out_stats_read(struct rte_pipeline * p,uint32_t port_id,struct rte_pipeline_port_out_stats * stats,int clear)1537 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1538 struct rte_pipeline_port_out_stats *stats, int clear)
1539 {
1540 struct rte_port_out *port;
1541 int retval;
1542
1543 if (p == NULL) {
1544 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1545 return -EINVAL;
1546 }
1547
1548 if (port_id >= p->num_ports_out) {
1549 RTE_LOG(ERR, PIPELINE,
1550 "%s: port OUT ID %u is out of range\n", __func__, port_id);
1551 return -EINVAL;
1552 }
1553
1554 port = &p->ports_out[port_id];
1555 if (port->ops.f_stats != NULL) {
1556 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1557 if (retval != 0)
1558 return retval;
1559 } else if (stats != NULL)
1560 memset(&stats->stats, 0, sizeof(stats->stats));
1561
1562 if (stats != NULL)
1563 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1564
1565 if (clear != 0)
1566 port->n_pkts_dropped_by_ah = 0;
1567
1568 return 0;
1569 }
1570
rte_pipeline_table_stats_read(struct rte_pipeline * p,uint32_t table_id,struct rte_pipeline_table_stats * stats,int clear)1571 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1572 struct rte_pipeline_table_stats *stats, int clear)
1573 {
1574 struct rte_table *table;
1575 int retval;
1576
1577 if (p == NULL) {
1578 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1579 __func__);
1580 return -EINVAL;
1581 }
1582
1583 if (table_id >= p->num_tables) {
1584 RTE_LOG(ERR, PIPELINE,
1585 "%s: table %u is out of range\n", __func__, table_id);
1586 return -EINVAL;
1587 }
1588
1589 table = &p->tables[table_id];
1590 if (table->ops.f_stats != NULL) {
1591 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1592 if (retval != 0)
1593 return retval;
1594 } else if (stats != NULL)
1595 memset(&stats->stats, 0, sizeof(stats->stats));
1596
1597 if (stats != NULL) {
1598 stats->n_pkts_dropped_by_lkp_hit_ah =
1599 table->n_pkts_dropped_by_lkp_hit_ah;
1600 stats->n_pkts_dropped_by_lkp_miss_ah =
1601 table->n_pkts_dropped_by_lkp_miss_ah;
1602 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1603 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1604 }
1605
1606 if (clear != 0) {
1607 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1608 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1609 table->n_pkts_dropped_lkp_hit = 0;
1610 table->n_pkts_dropped_lkp_miss = 0;
1611 }
1612
1613 return 0;
1614 }
1615