1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2015-2021 Atomic Rules LLC
3 */
4
5 #include <unistd.h>
6
7 #include "ark_ethdev_tx.h"
8 #include "ark_global.h"
9 #include "ark_mpu.h"
10 #include "ark_ddm.h"
11 #include "ark_logs.h"
12
13 #define ARK_TX_META_SIZE 32
14 #define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE)
15 #define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM)
16
17 #ifndef RTE_LIBRTE_ARK_MIN_TX_PKTLEN
18 #define ARK_MIN_TX_PKTLEN 0
19 #else
20 #define ARK_MIN_TX_PKTLEN RTE_LIBRTE_ARK_MIN_TX_PKTLEN
21 #endif
22
23 /* ************************************************************************* */
24 struct ark_tx_queue {
25 union ark_tx_meta *meta_q;
26 struct rte_mbuf **bufs;
27
28 /* handles for hw objects */
29 struct ark_mpu_t *mpu;
30 struct ark_ddm_t *ddm;
31
32 /* Stats HW tracks bytes and packets, need to count send errors */
33 uint64_t tx_errors;
34
35 tx_user_meta_hook_fn tx_user_meta_hook;
36 void *ext_user_data;
37
38 uint32_t queue_size;
39 uint32_t queue_mask;
40
41 /* 3 indexes to the paired data rings. */
42 int32_t prod_index; /* where to put the next one */
43 int32_t free_index; /* mbuf has been freed */
44
45 /* The queue Id is used to identify the HW Q */
46 uint16_t phys_qid;
47 /* The queue Index within the dpdk device structures */
48 uint16_t queue_index;
49
50 /* next cache line - fields written by device */
51 RTE_MARKER cacheline1 __rte_cache_min_aligned;
52 volatile int32_t cons_index; /* hw is done, can be freed */
53 } __rte_cache_aligned;
54
55 /* Forward declarations */
56 static int eth_ark_tx_jumbo(struct ark_tx_queue *queue,
57 struct rte_mbuf *mbuf,
58 uint32_t *user_meta, uint8_t meta_cnt);
59 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue);
60 static void free_completed_tx(struct ark_tx_queue *queue);
61
62 static inline void
ark_tx_hw_queue_stop(struct ark_tx_queue * queue)63 ark_tx_hw_queue_stop(struct ark_tx_queue *queue)
64 {
65 ark_mpu_stop(queue->mpu);
66 }
67
68 /* ************************************************************************* */
69 static inline void
eth_ark_tx_desc_fill(struct ark_tx_queue * queue,struct rte_mbuf * mbuf,uint8_t flags,uint32_t * user_meta,uint8_t meta_cnt)70 eth_ark_tx_desc_fill(struct ark_tx_queue *queue,
71 struct rte_mbuf *mbuf,
72 uint8_t flags,
73 uint32_t *user_meta,
74 uint8_t meta_cnt /* 0 to 5 */
75 )
76 {
77 uint32_t tx_idx;
78 union ark_tx_meta *meta;
79 uint8_t m;
80
81 /* Header */
82 tx_idx = queue->prod_index & queue->queue_mask;
83 meta = &queue->meta_q[tx_idx];
84 meta->data_len = rte_pktmbuf_data_len(mbuf);
85 meta->flags = flags;
86 meta->meta_cnt = meta_cnt / 2;
87 meta->user1 = meta_cnt ? (*user_meta++) : 0;
88 queue->prod_index++;
89
90 queue->bufs[tx_idx] = mbuf;
91
92 /* 1 or 2 user meta data entries, user words 1,2 and 3,4 */
93 for (m = 1; m < meta_cnt; m += 2) {
94 tx_idx = queue->prod_index & queue->queue_mask;
95 meta = &queue->meta_q[tx_idx];
96 meta->usermeta0 = *user_meta++;
97 meta->usermeta1 = *user_meta++;
98 queue->prod_index++;
99 }
100
101 tx_idx = queue->prod_index & queue->queue_mask;
102 meta = &queue->meta_q[tx_idx];
103 meta->physaddr = rte_mbuf_data_iova(mbuf);
104 queue->prod_index++;
105 }
106
107
108 /* ************************************************************************* */
109 uint16_t
eth_ark_xmit_pkts(void * vtxq,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)110 eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
111 {
112 struct ark_tx_queue *queue;
113 struct rte_mbuf *mbuf;
114 uint32_t user_meta[5];
115
116 int stat;
117 int32_t prod_index_limit;
118 uint16_t nb;
119 uint8_t user_len = 0;
120 const uint32_t min_pkt_len = ARK_MIN_TX_PKTLEN;
121 tx_user_meta_hook_fn tx_user_meta_hook;
122
123 queue = (struct ark_tx_queue *)vtxq;
124 tx_user_meta_hook = queue->tx_user_meta_hook;
125
126 /* free any packets after the HW is done with them */
127 free_completed_tx(queue);
128
129 /* leave 4 elements mpu data */
130 prod_index_limit = queue->queue_size + queue->free_index - 4;
131
132 for (nb = 0;
133 (nb < nb_pkts) && (prod_index_limit - queue->prod_index) > 0;
134 ++nb) {
135 mbuf = tx_pkts[nb];
136
137 if (min_pkt_len &&
138 unlikely(rte_pktmbuf_pkt_len(mbuf) < min_pkt_len)) {
139 /* this packet even if it is small can be split,
140 * be sure to add to the end mbuf
141 */
142 uint16_t to_add = min_pkt_len -
143 rte_pktmbuf_pkt_len(mbuf);
144 char *appended =
145 rte_pktmbuf_append(mbuf, to_add);
146
147 if (appended == 0) {
148 /* This packet is in error,
149 * we cannot send it so just
150 * count it and delete it.
151 */
152 queue->tx_errors += 1;
153 rte_pktmbuf_free(mbuf);
154 continue;
155 }
156 memset(appended, 0, to_add);
157 }
158
159 if (tx_user_meta_hook)
160 tx_user_meta_hook(mbuf, user_meta, &user_len,
161 queue->ext_user_data);
162 if (unlikely(mbuf->nb_segs != 1)) {
163 stat = eth_ark_tx_jumbo(queue, mbuf,
164 user_meta, user_len);
165 if (unlikely(stat != 0))
166 break; /* Queue is full */
167 } else {
168 eth_ark_tx_desc_fill(queue, mbuf,
169 ARK_DDM_SOP | ARK_DDM_EOP,
170 user_meta, user_len);
171 }
172 }
173
174 if (ARK_DEBUG_CORE && nb != nb_pkts) {
175 ARK_PMD_LOG(DEBUG, "TX: Failure to send:"
176 " req: %" PRIU32
177 " sent: %" PRIU32
178 " prod: %" PRIU32
179 " cons: %" PRIU32
180 " free: %" PRIU32 "\n",
181 nb_pkts, nb,
182 queue->prod_index,
183 queue->cons_index,
184 queue->free_index);
185 ark_mpu_dump(queue->mpu,
186 "TX Failure MPU: ",
187 queue->phys_qid);
188 }
189
190 /* let FPGA know producer index. */
191 if (likely(nb != 0))
192 ark_mpu_set_producer(queue->mpu, queue->prod_index);
193
194 return nb;
195 }
196
197 /* ************************************************************************* */
198 static int
eth_ark_tx_jumbo(struct ark_tx_queue * queue,struct rte_mbuf * mbuf,uint32_t * user_meta,uint8_t meta_cnt)199 eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf,
200 uint32_t *user_meta, uint8_t meta_cnt)
201 {
202 struct rte_mbuf *next;
203 int32_t free_queue_space;
204 uint8_t flags = ARK_DDM_SOP;
205
206 free_queue_space = queue->queue_mask -
207 (queue->prod_index - queue->free_index);
208 /* We need up to 4 mbufs for first header and 2 for subsequent ones */
209 if (unlikely(free_queue_space < (2 + (2 * mbuf->nb_segs))))
210 return -1;
211
212 while (mbuf != NULL) {
213 next = mbuf->next;
214 flags |= (next == NULL) ? ARK_DDM_EOP : 0;
215
216 eth_ark_tx_desc_fill(queue, mbuf, flags, user_meta, meta_cnt);
217
218 flags &= ~ARK_DDM_SOP; /* drop SOP flags */
219 meta_cnt = 0; /* Meta only on SOP */
220 mbuf = next;
221 }
222
223 return 0;
224 }
225
226 /* ************************************************************************* */
227 int
eth_ark_tx_queue_setup(struct rte_eth_dev * dev,uint16_t queue_idx,uint16_t nb_desc,unsigned int socket_id,const struct rte_eth_txconf * tx_conf __rte_unused)228 eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
229 uint16_t queue_idx,
230 uint16_t nb_desc,
231 unsigned int socket_id,
232 const struct rte_eth_txconf *tx_conf __rte_unused)
233 {
234 struct ark_adapter *ark = dev->data->dev_private;
235 struct ark_tx_queue *queue;
236 int status;
237
238 int qidx = queue_idx;
239
240 if (!rte_is_power_of_2(nb_desc)) {
241 ARK_PMD_LOG(ERR,
242 "DPDK Arkville configuration queue size"
243 " must be power of two %u (%s)\n",
244 nb_desc, __func__);
245 return -1;
246 }
247
248 /* Each packet requires at least 2 mpu elements - double desc count */
249 nb_desc = 2 * nb_desc;
250
251 /* Allocate queue struct */
252 queue = rte_zmalloc_socket("Ark_txqueue",
253 sizeof(struct ark_tx_queue),
254 64,
255 socket_id);
256 if (queue == 0) {
257 ARK_PMD_LOG(ERR, "Failed to allocate tx "
258 "queue memory in %s\n",
259 __func__);
260 return -ENOMEM;
261 }
262
263 /* we use zmalloc no need to initialize fields */
264 queue->queue_size = nb_desc;
265 queue->queue_mask = nb_desc - 1;
266 queue->phys_qid = qidx;
267 queue->queue_index = queue_idx;
268 dev->data->tx_queues[queue_idx] = queue;
269 queue->tx_user_meta_hook = ark->user_ext.tx_user_meta_hook;
270 queue->ext_user_data = ark->user_data[dev->data->port_id];
271
272 queue->meta_q =
273 rte_zmalloc_socket("Ark_txqueue meta",
274 nb_desc * sizeof(union ark_tx_meta),
275 64,
276 socket_id);
277 queue->bufs =
278 rte_zmalloc_socket("Ark_txqueue bufs",
279 nb_desc * sizeof(struct rte_mbuf *),
280 64,
281 socket_id);
282
283 if (queue->meta_q == 0 || queue->bufs == 0) {
284 ARK_PMD_LOG(ERR, "Failed to allocate "
285 "queue memory in %s\n", __func__);
286 rte_free(queue->meta_q);
287 rte_free(queue->bufs);
288 rte_free(queue);
289 return -ENOMEM;
290 }
291
292 queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET);
293 queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET);
294
295 status = eth_ark_tx_hw_queue_config(queue);
296
297 if (unlikely(status != 0)) {
298 rte_free(queue->meta_q);
299 rte_free(queue->bufs);
300 rte_free(queue);
301 return -1; /* ERROR CODE */
302 }
303
304 return 0;
305 }
306
307 /* ************************************************************************* */
308 static int
eth_ark_tx_hw_queue_config(struct ark_tx_queue * queue)309 eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue)
310 {
311 rte_iova_t queue_base, ring_base, cons_index_addr;
312 uint32_t write_interval_ns;
313
314 /* Verify HW -- MPU */
315 if (ark_mpu_verify(queue->mpu, sizeof(union ark_tx_meta)))
316 return -1;
317
318 queue_base = rte_malloc_virt2iova(queue);
319 ring_base = rte_malloc_virt2iova(queue->meta_q);
320 cons_index_addr =
321 queue_base + offsetof(struct ark_tx_queue, cons_index);
322
323 ark_mpu_stop(queue->mpu);
324 ark_mpu_reset(queue->mpu);
325
326 /* Stop and Reset and configure MPU */
327 ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1);
328
329 /*
330 * Adjust the write interval based on queue size --
331 * increase pcie traffic when low mbuf count
332 * Queue sizes less than 128 are not allowed
333 */
334 switch (queue->queue_size) {
335 case 128:
336 write_interval_ns = 500;
337 break;
338 case 256:
339 write_interval_ns = 500;
340 break;
341 case 512:
342 write_interval_ns = 1000;
343 break;
344 default:
345 write_interval_ns = 2000;
346 break;
347 }
348
349 /* Completion address in UDM */
350 ark_ddm_setup(queue->ddm, cons_index_addr, write_interval_ns);
351
352 return 0;
353 }
354
355 /* ************************************************************************* */
356 void
eth_ark_tx_queue_release(void * vtx_queue)357 eth_ark_tx_queue_release(void *vtx_queue)
358 {
359 struct ark_tx_queue *queue;
360
361 queue = (struct ark_tx_queue *)vtx_queue;
362
363 ark_tx_hw_queue_stop(queue);
364
365 queue->cons_index = queue->prod_index;
366 free_completed_tx(queue);
367
368 rte_free(queue->meta_q);
369 rte_free(queue->bufs);
370 rte_free(queue);
371 }
372
373 /* ************************************************************************* */
374 int
eth_ark_tx_queue_stop(struct rte_eth_dev * dev,uint16_t queue_id)375 eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
376 {
377 struct ark_tx_queue *queue;
378 int cnt = 0;
379
380 queue = dev->data->tx_queues[queue_id];
381
382 /* Wait for DDM to send out all packets. */
383 while (queue->cons_index != queue->prod_index) {
384 usleep(100);
385 if (cnt++ > 10000)
386 return -1;
387 }
388
389 ark_mpu_stop(queue->mpu);
390 free_completed_tx(queue);
391
392 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
393
394 return 0;
395 }
396
397 int
eth_ark_tx_queue_start(struct rte_eth_dev * dev,uint16_t queue_id)398 eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
399 {
400 struct ark_tx_queue *queue;
401
402 queue = dev->data->tx_queues[queue_id];
403 if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED)
404 return 0;
405
406 ark_mpu_start(queue->mpu);
407 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
408
409 return 0;
410 }
411
412 /* ************************************************************************* */
413 static void
free_completed_tx(struct ark_tx_queue * queue)414 free_completed_tx(struct ark_tx_queue *queue)
415 {
416 struct rte_mbuf *mbuf;
417 union ark_tx_meta *meta;
418 int32_t top_index;
419
420 top_index = queue->cons_index; /* read once */
421 while ((top_index - queue->free_index) > 0) {
422 meta = &queue->meta_q[queue->free_index & queue->queue_mask];
423 if (likely((meta->flags & ARK_DDM_SOP) != 0)) {
424 mbuf = queue->bufs[queue->free_index &
425 queue->queue_mask];
426 /* ref count of the mbuf is checked in this call. */
427 rte_pktmbuf_free(mbuf);
428 }
429 queue->free_index += (meta->meta_cnt + 2);
430 }
431 }
432
433 /* ************************************************************************* */
434 void
eth_tx_queue_stats_get(void * vqueue,struct rte_eth_stats * stats)435 eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats)
436 {
437 struct ark_tx_queue *queue;
438 struct ark_ddm_t *ddm;
439 uint64_t bytes, pkts;
440
441 queue = vqueue;
442 ddm = queue->ddm;
443
444 bytes = ark_ddm_queue_byte_count(ddm);
445 pkts = ark_ddm_queue_pkt_count(ddm);
446
447 stats->q_opackets[queue->queue_index] = pkts;
448 stats->q_obytes[queue->queue_index] = bytes;
449 stats->opackets += pkts;
450 stats->obytes += bytes;
451 stats->oerrors += queue->tx_errors;
452 }
453
454 void
eth_tx_queue_stats_reset(void * vqueue)455 eth_tx_queue_stats_reset(void *vqueue)
456 {
457 struct ark_tx_queue *queue;
458 struct ark_ddm_t *ddm;
459
460 queue = vqueue;
461 ddm = queue->ddm;
462
463 ark_ddm_queue_reset_stats(ddm);
464 queue->tx_errors = 0;
465 }
466