1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Cavium, Inc
3 */
4
5 #ifndef _TEST_PERF_COMMON_
6 #define _TEST_PERF_COMMON_
7
8 #include <stdio.h>
9 #include <stdbool.h>
10 #include <unistd.h>
11
12 #include <rte_cryptodev.h>
13 #include <rte_cycles.h>
14 #include <rte_ethdev.h>
15 #include <rte_event_crypto_adapter.h>
16 #include <rte_event_eth_rx_adapter.h>
17 #include <rte_event_eth_tx_adapter.h>
18 #include <rte_event_timer_adapter.h>
19 #include <rte_eventdev.h>
20 #include <rte_lcore.h>
21 #include <rte_malloc.h>
22 #include <rte_mempool.h>
23 #include <rte_prefetch.h>
24
25 #include "evt_common.h"
26 #include "evt_options.h"
27 #include "evt_test.h"
28
29 #define TEST_PERF_CA_ID 0
30
31 struct test_perf;
32
33 struct worker_data {
34 uint64_t processed_pkts;
35 uint64_t latency;
36 uint8_t dev_id;
37 uint8_t port_id;
38 struct test_perf *t;
39 } __rte_cache_aligned;
40
41 struct crypto_adptr_data {
42 uint8_t cdev_id;
43 uint16_t cdev_qp_id;
44 struct rte_cryptodev_sym_session **crypto_sess;
45 };
46 struct prod_data {
47 uint8_t dev_id;
48 uint8_t port_id;
49 uint8_t queue_id;
50 struct crypto_adptr_data ca;
51 struct test_perf *t;
52 } __rte_cache_aligned;
53
54 struct test_perf {
55 /* Don't change the offset of "done". Signal handler use this memory
56 * to terminate all lcores work.
57 */
58 int done;
59 uint64_t outstand_pkts;
60 uint8_t nb_workers;
61 enum evt_test_result result;
62 uint32_t nb_flows;
63 uint64_t nb_pkts;
64 struct rte_mempool *pool;
65 struct prod_data prod[EVT_MAX_PORTS];
66 struct worker_data worker[EVT_MAX_PORTS];
67 struct evt_options *opt;
68 uint8_t sched_type_list[EVT_MAX_STAGES] __rte_cache_aligned;
69 struct rte_event_timer_adapter *timer_adptr[
70 RTE_EVENT_TIMER_ADAPTER_NUM_MAX] __rte_cache_aligned;
71 struct rte_mempool *ca_op_pool;
72 struct rte_mempool *ca_sess_pool;
73 struct rte_mempool *ca_sess_priv_pool;
74 } __rte_cache_aligned;
75
76 struct perf_elt {
77 union {
78 struct rte_event_timer tim;
79 struct {
80 char pad[offsetof(struct rte_event_timer, user_meta)];
81 uint64_t timestamp;
82 };
83 };
84 } __rte_cache_aligned;
85
86 #define BURST_SIZE 16
87 #define MAX_PROD_ENQ_BURST_SIZE 128
88
89 #define PERF_WORKER_INIT\
90 struct worker_data *w = arg;\
91 struct test_perf *t = w->t;\
92 struct evt_options *opt = t->opt;\
93 const uint8_t dev = w->dev_id;\
94 const uint8_t port = w->port_id;\
95 const uint8_t prod_timer_type = \
96 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR;\
97 const uint8_t prod_crypto_type = \
98 opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR;\
99 uint8_t *const sched_type_list = &t->sched_type_list[0];\
100 struct rte_mempool *const pool = t->pool;\
101 const uint8_t nb_stages = t->opt->nb_stages;\
102 const uint8_t laststage = nb_stages - 1;\
103 uint8_t cnt = 0;\
104 void *bufs[16] __rte_cache_aligned;\
105 int const sz = RTE_DIM(bufs);\
106 if (opt->verbose_level > 1)\
107 printf("%s(): lcore %d dev_id %d port=%d\n", __func__,\
108 rte_lcore_id(), dev, port)
109
110 static __rte_always_inline int
perf_process_last_stage(struct rte_mempool * const pool,struct rte_event * const ev,struct worker_data * const w,void * bufs[],int const buf_sz,uint8_t count)111 perf_process_last_stage(struct rte_mempool *const pool,
112 struct rte_event *const ev, struct worker_data *const w,
113 void *bufs[], int const buf_sz, uint8_t count)
114 {
115 bufs[count++] = ev->event_ptr;
116
117 /* release fence here ensures event_prt is
118 * stored before updating the number of
119 * processed packets for worker lcores
120 */
121 rte_atomic_thread_fence(__ATOMIC_RELEASE);
122 w->processed_pkts++;
123
124 if (unlikely(count == buf_sz)) {
125 count = 0;
126 rte_mempool_put_bulk(pool, bufs, buf_sz);
127 }
128 return count;
129 }
130
131 static __rte_always_inline uint8_t
perf_process_last_stage_latency(struct rte_mempool * const pool,struct rte_event * const ev,struct worker_data * const w,void * bufs[],int const buf_sz,uint8_t count)132 perf_process_last_stage_latency(struct rte_mempool *const pool,
133 struct rte_event *const ev, struct worker_data *const w,
134 void *bufs[], int const buf_sz, uint8_t count)
135 {
136 uint64_t latency;
137 struct perf_elt *const m = ev->event_ptr;
138
139 bufs[count++] = ev->event_ptr;
140
141 /* release fence here ensures event_prt is
142 * stored before updating the number of
143 * processed packets for worker lcores
144 */
145 rte_atomic_thread_fence(__ATOMIC_RELEASE);
146 w->processed_pkts++;
147
148 if (unlikely(count == buf_sz)) {
149 count = 0;
150 latency = rte_get_timer_cycles() - m->timestamp;
151 rte_mempool_put_bulk(pool, bufs, buf_sz);
152 } else {
153 latency = rte_get_timer_cycles() - m->timestamp;
154 }
155
156 w->latency += latency;
157 return count;
158 }
159
160
161 static inline int
perf_nb_event_ports(struct evt_options * opt)162 perf_nb_event_ports(struct evt_options *opt)
163 {
164 return evt_nr_active_lcores(opt->wlcores) +
165 evt_nr_active_lcores(opt->plcores);
166 }
167
168 int perf_test_result(struct evt_test *test, struct evt_options *opt);
169 int perf_opt_check(struct evt_options *opt, uint64_t nb_queues);
170 int perf_test_setup(struct evt_test *test, struct evt_options *opt);
171 int perf_ethdev_setup(struct evt_test *test, struct evt_options *opt);
172 int perf_cryptodev_setup(struct evt_test *test, struct evt_options *opt);
173 int perf_mempool_setup(struct evt_test *test, struct evt_options *opt);
174 int perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
175 uint8_t stride, uint8_t nb_queues,
176 const struct rte_event_port_conf *port_conf);
177 int perf_event_dev_service_setup(uint8_t dev_id);
178 int perf_launch_lcores(struct evt_test *test, struct evt_options *opt,
179 int (*worker)(void *));
180 void perf_opt_dump(struct evt_options *opt, uint8_t nb_queues);
181 void perf_test_destroy(struct evt_test *test, struct evt_options *opt);
182 void perf_eventdev_destroy(struct evt_test *test, struct evt_options *opt);
183 void perf_cryptodev_destroy(struct evt_test *test, struct evt_options *opt);
184 void perf_ethdev_destroy(struct evt_test *test, struct evt_options *opt);
185 void perf_ethdev_rx_stop(struct evt_test *test, struct evt_options *opt);
186 void perf_mempool_destroy(struct evt_test *test, struct evt_options *opt);
187 void perf_worker_cleanup(struct rte_mempool *const pool, uint8_t dev_id,
188 uint8_t port_id, struct rte_event events[],
189 uint16_t nb_enq, uint16_t nb_deq);
190
191 #endif /* _TEST_PERF_COMMON_ */
192