1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
3 */
4
5 #ifndef __OTX2_EVDEV_H__
6 #define __OTX2_EVDEV_H__
7
8 #include <rte_eventdev.h>
9 #include <rte_eventdev_pmd.h>
10 #include <rte_event_eth_rx_adapter.h>
11 #include <rte_event_eth_tx_adapter.h>
12
13 #include "otx2_common.h"
14 #include "otx2_dev.h"
15 #include "otx2_ethdev.h"
16 #include "otx2_mempool.h"
17 #include "otx2_tim_evdev.h"
18
19 #define EVENTDEV_NAME_OCTEONTX2_PMD event_octeontx2
20
21 #define sso_func_trace otx2_sso_dbg
22
23 #define OTX2_SSO_MAX_VHGRP RTE_EVENT_MAX_QUEUES_PER_DEV
24 #define OTX2_SSO_MAX_VHWS (UINT8_MAX)
25 #define OTX2_SSO_FC_NAME "otx2_evdev_xaq_fc"
26 #define OTX2_SSO_SQB_LIMIT (0x180)
27 #define OTX2_SSO_XAQ_SLACK (8)
28 #define OTX2_SSO_XAQ_CACHE_CNT (0x7)
29 #define OTX2_SSO_WQE_SG_PTR (9)
30
31 /* SSO LF register offsets (BAR2) */
32 #define SSO_LF_GGRP_OP_ADD_WORK0 (0x0ull)
33 #define SSO_LF_GGRP_OP_ADD_WORK1 (0x8ull)
34
35 #define SSO_LF_GGRP_QCTL (0x20ull)
36 #define SSO_LF_GGRP_EXE_DIS (0x80ull)
37 #define SSO_LF_GGRP_INT (0x100ull)
38 #define SSO_LF_GGRP_INT_W1S (0x108ull)
39 #define SSO_LF_GGRP_INT_ENA_W1S (0x110ull)
40 #define SSO_LF_GGRP_INT_ENA_W1C (0x118ull)
41 #define SSO_LF_GGRP_INT_THR (0x140ull)
42 #define SSO_LF_GGRP_INT_CNT (0x180ull)
43 #define SSO_LF_GGRP_XAQ_CNT (0x1b0ull)
44 #define SSO_LF_GGRP_AQ_CNT (0x1c0ull)
45 #define SSO_LF_GGRP_AQ_THR (0x1e0ull)
46 #define SSO_LF_GGRP_MISC_CNT (0x200ull)
47
48 /* SSOW LF register offsets (BAR2) */
49 #define SSOW_LF_GWS_LINKS (0x10ull)
50 #define SSOW_LF_GWS_PENDWQP (0x40ull)
51 #define SSOW_LF_GWS_PENDSTATE (0x50ull)
52 #define SSOW_LF_GWS_NW_TIM (0x70ull)
53 #define SSOW_LF_GWS_GRPMSK_CHG (0x80ull)
54 #define SSOW_LF_GWS_INT (0x100ull)
55 #define SSOW_LF_GWS_INT_W1S (0x108ull)
56 #define SSOW_LF_GWS_INT_ENA_W1S (0x110ull)
57 #define SSOW_LF_GWS_INT_ENA_W1C (0x118ull)
58 #define SSOW_LF_GWS_TAG (0x200ull)
59 #define SSOW_LF_GWS_WQP (0x210ull)
60 #define SSOW_LF_GWS_SWTP (0x220ull)
61 #define SSOW_LF_GWS_PENDTAG (0x230ull)
62 #define SSOW_LF_GWS_OP_ALLOC_WE (0x400ull)
63 #define SSOW_LF_GWS_OP_GET_WORK (0x600ull)
64 #define SSOW_LF_GWS_OP_SWTAG_FLUSH (0x800ull)
65 #define SSOW_LF_GWS_OP_SWTAG_UNTAG (0x810ull)
66 #define SSOW_LF_GWS_OP_SWTP_CLR (0x820ull)
67 #define SSOW_LF_GWS_OP_UPD_WQP_GRP0 (0x830ull)
68 #define SSOW_LF_GWS_OP_UPD_WQP_GRP1 (0x838ull)
69 #define SSOW_LF_GWS_OP_DESCHED (0x880ull)
70 #define SSOW_LF_GWS_OP_DESCHED_NOSCH (0x8c0ull)
71 #define SSOW_LF_GWS_OP_SWTAG_DESCHED (0x980ull)
72 #define SSOW_LF_GWS_OP_SWTAG_NOSCHED (0x9c0ull)
73 #define SSOW_LF_GWS_OP_CLR_NSCHED0 (0xa00ull)
74 #define SSOW_LF_GWS_OP_CLR_NSCHED1 (0xa08ull)
75 #define SSOW_LF_GWS_OP_SWTP_SET (0xc00ull)
76 #define SSOW_LF_GWS_OP_SWTAG_NORM (0xc10ull)
77 #define SSOW_LF_GWS_OP_SWTAG_FULL0 (0xc20ull)
78 #define SSOW_LF_GWS_OP_SWTAG_FULL1 (0xc28ull)
79 #define SSOW_LF_GWS_OP_GWC_INVAL (0xe00ull)
80
81 #define OTX2_SSOW_GET_BASE_ADDR(_GW) ((_GW) - SSOW_LF_GWS_OP_GET_WORK)
82 #define OTX2_SSOW_TT_FROM_TAG(x) (((x) >> 32) & SSO_TT_EMPTY)
83
84 #define NSEC2USEC(__ns) ((__ns) / 1E3)
85 #define USEC2NSEC(__us) ((__us) * 1E3)
86 #define NSEC2TICK(__ns, __freq) (((__ns) * (__freq)) / 1E9)
87 #define TICK2NSEC(__tck, __freq) (((__tck) * 1E9) / (__freq))
88
89 enum otx2_sso_lf_type {
90 SSO_LF_GGRP,
91 SSO_LF_GWS
92 };
93
94 union otx2_sso_event {
95 uint64_t get_work0;
96 struct {
97 uint32_t flow_id:20;
98 uint32_t sub_event_type:8;
99 uint32_t event_type:4;
100 uint8_t op:2;
101 uint8_t rsvd:4;
102 uint8_t sched_type:2;
103 uint8_t queue_id;
104 uint8_t priority;
105 uint8_t impl_opaque;
106 };
107 } __rte_aligned(64);
108
109 enum {
110 SSO_SYNC_ORDERED,
111 SSO_SYNC_ATOMIC,
112 SSO_SYNC_UNTAGGED,
113 SSO_SYNC_EMPTY
114 };
115
116 struct otx2_sso_qos {
117 uint8_t queue;
118 uint8_t xaq_prcnt;
119 uint8_t taq_prcnt;
120 uint8_t iaq_prcnt;
121 };
122
123 struct otx2_sso_evdev {
124 OTX2_DEV; /* Base class */
125 uint8_t max_event_queues;
126 uint8_t max_event_ports;
127 uint8_t is_timeout_deq;
128 uint8_t nb_event_queues;
129 uint8_t nb_event_ports;
130 uint8_t configured;
131 uint32_t deq_tmo_ns;
132 uint32_t min_dequeue_timeout_ns;
133 uint32_t max_dequeue_timeout_ns;
134 int32_t max_num_events;
135 uint64_t *fc_mem;
136 uint64_t xaq_lmt;
137 uint64_t nb_xaq_cfg;
138 rte_iova_t fc_iova;
139 struct rte_mempool *xaq_pool;
140 uint64_t rx_offloads;
141 uint64_t tx_offloads;
142 uint64_t adptr_xae_cnt;
143 uint16_t rx_adptr_pool_cnt;
144 uint64_t *rx_adptr_pools;
145 uint16_t max_port_id;
146 uint16_t tim_adptr_ring_cnt;
147 uint16_t *timer_adptr_rings;
148 uint64_t *timer_adptr_sz;
149 /* Dev args */
150 uint8_t dual_ws;
151 uint32_t xae_cnt;
152 uint8_t qos_queue_cnt;
153 struct otx2_sso_qos *qos_parse_data;
154 /* HW const */
155 uint32_t xae_waes;
156 uint32_t xaq_buf_size;
157 uint32_t iue;
158 /* MSIX offsets */
159 uint16_t sso_msixoff[OTX2_SSO_MAX_VHGRP];
160 uint16_t ssow_msixoff[OTX2_SSO_MAX_VHWS];
161 /* PTP timestamp */
162 struct otx2_timesync_info *tstamp;
163 } __rte_cache_aligned;
164
165 #define OTX2_SSOGWS_OPS \
166 /* WS ops */ \
167 uintptr_t getwrk_op; \
168 uintptr_t tag_op; \
169 uintptr_t wqp_op; \
170 uintptr_t swtag_flush_op; \
171 uintptr_t swtag_norm_op; \
172 uintptr_t swtag_desched_op; \
173 uint8_t cur_tt; \
174 uint8_t cur_grp
175
176 /* Event port aka GWS */
177 struct otx2_ssogws {
178 /* Get Work Fastpath data */
179 OTX2_SSOGWS_OPS;
180 uint8_t swtag_req;
181 void *lookup_mem;
182 uint8_t port;
183 /* Add Work Fastpath data */
184 uint64_t xaq_lmt __rte_cache_aligned;
185 uint64_t *fc_mem;
186 uintptr_t grps_base[OTX2_SSO_MAX_VHGRP];
187 /* PTP timestamp */
188 struct otx2_timesync_info *tstamp;
189 /* Tx Fastpath data */
190 uint8_t tx_adptr_data[] __rte_cache_aligned;
191 } __rte_cache_aligned;
192
193 struct otx2_ssogws_state {
194 OTX2_SSOGWS_OPS;
195 };
196
197 struct otx2_ssogws_dual {
198 /* Get Work Fastpath data */
199 struct otx2_ssogws_state ws_state[2]; /* Ping and Pong */
200 uint8_t swtag_req;
201 uint8_t vws; /* Ping pong bit */
202 void *lookup_mem;
203 uint8_t port;
204 /* Add Work Fastpath data */
205 uint64_t xaq_lmt __rte_cache_aligned;
206 uint64_t *fc_mem;
207 uintptr_t grps_base[OTX2_SSO_MAX_VHGRP];
208 /* PTP timestamp */
209 struct otx2_timesync_info *tstamp;
210 /* Tx Fastpath data */
211 uint8_t tx_adptr_data[] __rte_cache_aligned;
212 } __rte_cache_aligned;
213
214 static inline struct otx2_sso_evdev *
sso_pmd_priv(const struct rte_eventdev * event_dev)215 sso_pmd_priv(const struct rte_eventdev *event_dev)
216 {
217 return event_dev->data->dev_private;
218 }
219
220 struct otx2_ssogws_cookie {
221 const struct rte_eventdev *event_dev;
222 bool configured;
223 };
224
225 static inline struct otx2_ssogws_cookie *
ssogws_get_cookie(void * ws)226 ssogws_get_cookie(void *ws)
227 {
228 return (struct otx2_ssogws_cookie *)
229 ((uint8_t *)ws - RTE_CACHE_LINE_SIZE);
230 }
231
232 static const union mbuf_initializer mbuf_init = {
233 .fields = {
234 .data_off = RTE_PKTMBUF_HEADROOM,
235 .refcnt = 1,
236 .nb_segs = 1,
237 .port = 0
238 }
239 };
240
241 static __rte_always_inline void
otx2_wqe_to_mbuf(uint64_t get_work1,const uint64_t mbuf,uint8_t port_id,const uint32_t tag,const uint32_t flags,const void * const lookup_mem)242 otx2_wqe_to_mbuf(uint64_t get_work1, const uint64_t mbuf, uint8_t port_id,
243 const uint32_t tag, const uint32_t flags,
244 const void * const lookup_mem)
245 {
246 struct nix_wqe_hdr_s *wqe = (struct nix_wqe_hdr_s *)get_work1;
247 uint64_t val = mbuf_init.value | (uint64_t)port_id << 48;
248
249 if (flags & NIX_RX_OFFLOAD_TSTAMP_F)
250 val |= NIX_TIMESYNC_RX_OFFSET;
251
252 otx2_nix_cqe_to_mbuf((struct nix_cqe_hdr_s *)wqe, tag,
253 (struct rte_mbuf *)mbuf, lookup_mem,
254 val, flags);
255
256 }
257
258 static inline int
parse_kvargs_flag(const char * key,const char * value,void * opaque)259 parse_kvargs_flag(const char *key, const char *value, void *opaque)
260 {
261 RTE_SET_USED(key);
262
263 *(uint8_t *)opaque = !!atoi(value);
264 return 0;
265 }
266
267 static inline int
parse_kvargs_value(const char * key,const char * value,void * opaque)268 parse_kvargs_value(const char *key, const char *value, void *opaque)
269 {
270 RTE_SET_USED(key);
271
272 *(uint32_t *)opaque = (uint32_t)atoi(value);
273 return 0;
274 }
275
276 #define SSO_RX_ADPTR_ENQ_FASTPATH_FUNC NIX_RX_FASTPATH_MODES
277 #define SSO_TX_ADPTR_ENQ_FASTPATH_FUNC NIX_TX_FASTPATH_MODES
278
279 /* Single WS API's */
280 uint16_t otx2_ssogws_enq(void *port, const struct rte_event *ev);
281 uint16_t otx2_ssogws_enq_burst(void *port, const struct rte_event ev[],
282 uint16_t nb_events);
283 uint16_t otx2_ssogws_enq_new_burst(void *port, const struct rte_event ev[],
284 uint16_t nb_events);
285 uint16_t otx2_ssogws_enq_fwd_burst(void *port, const struct rte_event ev[],
286 uint16_t nb_events);
287
288 /* Dual WS API's */
289 uint16_t otx2_ssogws_dual_enq(void *port, const struct rte_event *ev);
290 uint16_t otx2_ssogws_dual_enq_burst(void *port, const struct rte_event ev[],
291 uint16_t nb_events);
292 uint16_t otx2_ssogws_dual_enq_new_burst(void *port, const struct rte_event ev[],
293 uint16_t nb_events);
294 uint16_t otx2_ssogws_dual_enq_fwd_burst(void *port, const struct rte_event ev[],
295 uint16_t nb_events);
296
297 /* Auto generated API's */
298 #define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
299 uint16_t otx2_ssogws_deq_ ##name(void *port, struct rte_event *ev, \
300 uint64_t timeout_ticks); \
301 uint16_t otx2_ssogws_deq_burst_ ##name(void *port, struct rte_event ev[], \
302 uint16_t nb_events, \
303 uint64_t timeout_ticks); \
304 uint16_t otx2_ssogws_deq_timeout_ ##name(void *port, \
305 struct rte_event *ev, \
306 uint64_t timeout_ticks); \
307 uint16_t otx2_ssogws_deq_timeout_burst_ ##name(void *port, \
308 struct rte_event ev[], \
309 uint16_t nb_events, \
310 uint64_t timeout_ticks); \
311 uint16_t otx2_ssogws_deq_seg_ ##name(void *port, struct rte_event *ev, \
312 uint64_t timeout_ticks); \
313 uint16_t otx2_ssogws_deq_seg_burst_ ##name(void *port, \
314 struct rte_event ev[], \
315 uint16_t nb_events, \
316 uint64_t timeout_ticks); \
317 uint16_t otx2_ssogws_deq_seg_timeout_ ##name(void *port, \
318 struct rte_event *ev, \
319 uint64_t timeout_ticks); \
320 uint16_t otx2_ssogws_deq_seg_timeout_burst_ ##name(void *port, \
321 struct rte_event ev[], \
322 uint16_t nb_events, \
323 uint64_t timeout_ticks); \
324 \
325 uint16_t otx2_ssogws_dual_deq_ ##name(void *port, struct rte_event *ev, \
326 uint64_t timeout_ticks); \
327 uint16_t otx2_ssogws_dual_deq_burst_ ##name(void *port, \
328 struct rte_event ev[], \
329 uint16_t nb_events, \
330 uint64_t timeout_ticks); \
331 uint16_t otx2_ssogws_dual_deq_timeout_ ##name(void *port, \
332 struct rte_event *ev, \
333 uint64_t timeout_ticks); \
334 uint16_t otx2_ssogws_dual_deq_timeout_burst_ ##name(void *port, \
335 struct rte_event ev[], \
336 uint16_t nb_events, \
337 uint64_t timeout_ticks); \
338 uint16_t otx2_ssogws_dual_deq_seg_ ##name(void *port, struct rte_event *ev, \
339 uint64_t timeout_ticks); \
340 uint16_t otx2_ssogws_dual_deq_seg_burst_ ##name(void *port, \
341 struct rte_event ev[], \
342 uint16_t nb_events, \
343 uint64_t timeout_ticks); \
344 uint16_t otx2_ssogws_dual_deq_seg_timeout_ ##name(void *port, \
345 struct rte_event *ev, \
346 uint64_t timeout_ticks); \
347 uint16_t otx2_ssogws_dual_deq_seg_timeout_burst_ ##name(void *port, \
348 struct rte_event ev[], \
349 uint16_t nb_events, \
350 uint64_t timeout_ticks);\
351
352 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
353 #undef R
354
355 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \
356 uint16_t otx2_ssogws_tx_adptr_enq_ ## name(void *port, struct rte_event ev[],\
357 uint16_t nb_events); \
358 uint16_t otx2_ssogws_tx_adptr_enq_seg_ ## name(void *port, \
359 struct rte_event ev[], \
360 uint16_t nb_events); \
361 uint16_t otx2_ssogws_dual_tx_adptr_enq_ ## name(void *port, \
362 struct rte_event ev[], \
363 uint16_t nb_events); \
364 uint16_t otx2_ssogws_dual_tx_adptr_enq_seg_ ## name(void *port, \
365 struct rte_event ev[], \
366 uint16_t nb_events); \
367
368 SSO_TX_ADPTR_ENQ_FASTPATH_FUNC
369 #undef T
370
371 void sso_updt_xae_cnt(struct otx2_sso_evdev *dev, void *data,
372 uint32_t event_type);
373 int sso_xae_reconfigure(struct rte_eventdev *event_dev);
374 void sso_fastpath_fns_set(struct rte_eventdev *event_dev);
375
376 int otx2_sso_rx_adapter_caps_get(const struct rte_eventdev *event_dev,
377 const struct rte_eth_dev *eth_dev,
378 uint32_t *caps);
379 int otx2_sso_rx_adapter_queue_add(const struct rte_eventdev *event_dev,
380 const struct rte_eth_dev *eth_dev,
381 int32_t rx_queue_id,
382 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf);
383 int otx2_sso_rx_adapter_queue_del(const struct rte_eventdev *event_dev,
384 const struct rte_eth_dev *eth_dev,
385 int32_t rx_queue_id);
386 int otx2_sso_rx_adapter_start(const struct rte_eventdev *event_dev,
387 const struct rte_eth_dev *eth_dev);
388 int otx2_sso_rx_adapter_stop(const struct rte_eventdev *event_dev,
389 const struct rte_eth_dev *eth_dev);
390 int otx2_sso_tx_adapter_caps_get(const struct rte_eventdev *dev,
391 const struct rte_eth_dev *eth_dev,
392 uint32_t *caps);
393 int otx2_sso_tx_adapter_queue_add(uint8_t id,
394 const struct rte_eventdev *event_dev,
395 const struct rte_eth_dev *eth_dev,
396 int32_t tx_queue_id);
397
398 int otx2_sso_tx_adapter_queue_del(uint8_t id,
399 const struct rte_eventdev *event_dev,
400 const struct rte_eth_dev *eth_dev,
401 int32_t tx_queue_id);
402
403 /* Event crypto adapter API's */
404 int otx2_ca_caps_get(const struct rte_eventdev *dev,
405 const struct rte_cryptodev *cdev, uint32_t *caps);
406
407 int otx2_ca_qp_add(const struct rte_eventdev *dev,
408 const struct rte_cryptodev *cdev, int32_t queue_pair_id,
409 const struct rte_event *event);
410
411 int otx2_ca_qp_del(const struct rte_eventdev *dev,
412 const struct rte_cryptodev *cdev, int32_t queue_pair_id);
413
414 /* Clean up API's */
415 typedef void (*otx2_handle_event_t)(void *arg, struct rte_event ev);
416 void ssogws_flush_events(struct otx2_ssogws *ws, uint8_t queue_id,
417 uintptr_t base, otx2_handle_event_t fn, void *arg);
418 void ssogws_reset(struct otx2_ssogws *ws);
419 /* Selftest */
420 int otx2_sso_selftest(void);
421 /* Init and Fini API's */
422 int otx2_sso_init(struct rte_eventdev *event_dev);
423 int otx2_sso_fini(struct rte_eventdev *event_dev);
424 /* IRQ handlers */
425 int sso_register_irqs(const struct rte_eventdev *event_dev);
426 void sso_unregister_irqs(const struct rte_eventdev *event_dev);
427
428 #endif /* __OTX2_EVDEV_H__ */
429