1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019-2020 Intel Corporation
3 */
4 #ifndef _RTE_IOAT_RAWDEV_FNS_H_
5 #define _RTE_IOAT_RAWDEV_FNS_H_
6
7 /**
8 * @file
9 * This header file contains the implementation of the various ioat
10 * rawdev functions for IOAT/CBDMA hardware. The API specification and key
11 * public structures are defined in "rte_ioat_rawdev.h".
12 *
13 * This file should not be included directly, but instead applications should
14 * include "rte_ioat_rawdev.h", which then includes this file - and the IDXD/DSA
15 * equivalent header - in turn.
16 */
17
18 #include <x86intrin.h>
19 #include <rte_rawdev.h>
20 #include <rte_memzone.h>
21 #include <rte_prefetch.h>
22
23 /**
24 * @internal
25 * Identify the data path to use.
26 * Must be first field of rte_ioat_rawdev and rte_idxd_rawdev structs
27 */
28 enum rte_ioat_dev_type {
29 RTE_IOAT_DEV,
30 RTE_IDXD_DEV,
31 };
32
33 /**
34 * @internal
35 * some statistics for tracking, if added/changed update xstats fns
36 */
37 struct rte_ioat_xstats {
38 uint64_t enqueue_failed;
39 uint64_t enqueued;
40 uint64_t started;
41 uint64_t completed;
42 };
43
44 #include "rte_idxd_rawdev_fns.h"
45
46 /**
47 * @internal
48 * Structure representing a device descriptor
49 */
50 struct rte_ioat_generic_hw_desc {
51 uint32_t size;
52 union {
53 uint32_t control_raw;
54 struct {
55 uint32_t int_enable: 1;
56 uint32_t src_snoop_disable: 1;
57 uint32_t dest_snoop_disable: 1;
58 uint32_t completion_update: 1;
59 uint32_t fence: 1;
60 uint32_t reserved2: 1;
61 uint32_t src_page_break: 1;
62 uint32_t dest_page_break: 1;
63 uint32_t bundle: 1;
64 uint32_t dest_dca: 1;
65 uint32_t hint: 1;
66 uint32_t reserved: 13;
67 uint32_t op: 8;
68 } control;
69 } u;
70 uint64_t src_addr;
71 uint64_t dest_addr;
72 uint64_t next;
73 uint64_t op_specific[4];
74 };
75
76 /**
77 * @internal
78 * Structure representing an IOAT device instance
79 */
80 struct rte_ioat_rawdev {
81 /* common fields at the top - match those in rte_idxd_rawdev */
82 enum rte_ioat_dev_type type;
83 struct rte_ioat_xstats xstats;
84
85 struct rte_rawdev *rawdev;
86 const struct rte_memzone *mz;
87 const struct rte_memzone *desc_mz;
88
89 volatile uint16_t *doorbell __rte_cache_aligned;
90 phys_addr_t status_addr;
91 phys_addr_t ring_addr;
92
93 unsigned short ring_size;
94 bool hdls_disable;
95 struct rte_ioat_generic_hw_desc *desc_ring;
96 __m128i *hdls; /* completion handles for returning to user */
97
98
99 unsigned short next_read;
100 unsigned short next_write;
101
102 /* to report completions, the device will write status back here */
103 volatile uint64_t status __rte_cache_aligned;
104
105 /* pointer to the register bar */
106 volatile struct rte_ioat_registers *regs;
107 };
108
109 #define RTE_IOAT_CHANSTS_IDLE 0x1
110 #define RTE_IOAT_CHANSTS_SUSPENDED 0x2
111 #define RTE_IOAT_CHANSTS_HALTED 0x3
112 #define RTE_IOAT_CHANSTS_ARMED 0x4
113
114 static __rte_always_inline uint16_t
__ioat_burst_capacity(int dev_id)115 __ioat_burst_capacity(int dev_id)
116 {
117 struct rte_ioat_rawdev *ioat =
118 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
119 unsigned short size = ioat->ring_size - 1;
120 unsigned short read = ioat->next_read;
121 unsigned short write = ioat->next_write;
122 unsigned short space = size - (write - read);
123
124 return space;
125 }
126
127 static __rte_always_inline int
__ioat_write_desc(int dev_id,uint32_t op,uint64_t src,phys_addr_t dst,unsigned int length,uintptr_t src_hdl,uintptr_t dst_hdl)128 __ioat_write_desc(int dev_id, uint32_t op, uint64_t src, phys_addr_t dst,
129 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
130 {
131 struct rte_ioat_rawdev *ioat =
132 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
133 unsigned short read = ioat->next_read;
134 unsigned short write = ioat->next_write;
135 unsigned short mask = ioat->ring_size - 1;
136 unsigned short space = mask + read - write;
137 struct rte_ioat_generic_hw_desc *desc;
138
139 if (space == 0) {
140 ioat->xstats.enqueue_failed++;
141 return 0;
142 }
143
144 ioat->next_write = write + 1;
145 write &= mask;
146
147 desc = &ioat->desc_ring[write];
148 desc->size = length;
149 /* set descriptor write-back every 16th descriptor */
150 desc->u.control_raw = (uint32_t)((op << IOAT_CMD_OP_SHIFT) |
151 (!(write & 0xF) << IOAT_COMP_UPDATE_SHIFT));
152 desc->src_addr = src;
153 desc->dest_addr = dst;
154
155 if (!ioat->hdls_disable)
156 ioat->hdls[write] = _mm_set_epi64x((int64_t)dst_hdl,
157 (int64_t)src_hdl);
158 rte_prefetch0(&ioat->desc_ring[ioat->next_write & mask]);
159
160 ioat->xstats.enqueued++;
161 return 1;
162 }
163
164 static __rte_always_inline int
__ioat_enqueue_fill(int dev_id,uint64_t pattern,phys_addr_t dst,unsigned int length,uintptr_t dst_hdl)165 __ioat_enqueue_fill(int dev_id, uint64_t pattern, phys_addr_t dst,
166 unsigned int length, uintptr_t dst_hdl)
167 {
168 static const uintptr_t null_hdl;
169
170 return __ioat_write_desc(dev_id, ioat_op_fill, pattern, dst, length,
171 null_hdl, dst_hdl);
172 }
173
174 /*
175 * Enqueue a copy operation onto the ioat device
176 */
177 static __rte_always_inline int
__ioat_enqueue_copy(int dev_id,phys_addr_t src,phys_addr_t dst,unsigned int length,uintptr_t src_hdl,uintptr_t dst_hdl)178 __ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst,
179 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
180 {
181 return __ioat_write_desc(dev_id, ioat_op_copy, src, dst, length,
182 src_hdl, dst_hdl);
183 }
184
185 /* add fence to last written descriptor */
186 static __rte_always_inline int
__ioat_fence(int dev_id)187 __ioat_fence(int dev_id)
188 {
189 struct rte_ioat_rawdev *ioat =
190 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
191 unsigned short write = ioat->next_write;
192 unsigned short mask = ioat->ring_size - 1;
193 struct rte_ioat_generic_hw_desc *desc;
194
195 write = (write - 1) & mask;
196 desc = &ioat->desc_ring[write];
197
198 desc->u.control.fence = 1;
199 return 0;
200 }
201
202 /*
203 * Trigger hardware to begin performing enqueued operations
204 */
205 static __rte_always_inline int
__ioat_perform_ops(int dev_id)206 __ioat_perform_ops(int dev_id)
207 {
208 struct rte_ioat_rawdev *ioat =
209 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
210 ioat->desc_ring[(ioat->next_write - 1) & (ioat->ring_size - 1)].u
211 .control.completion_update = 1;
212 rte_compiler_barrier();
213 *ioat->doorbell = ioat->next_write;
214 ioat->xstats.started = ioat->xstats.enqueued;
215
216 return 0;
217 }
218
219 /**
220 * @internal
221 * Returns the index of the last completed operation.
222 */
223 static __rte_always_inline int
__ioat_get_last_completed(struct rte_ioat_rawdev * ioat,int * error)224 __ioat_get_last_completed(struct rte_ioat_rawdev *ioat, int *error)
225 {
226 uint64_t status = ioat->status;
227
228 /* lower 3 bits indicate "transfer status" : active, idle, halted.
229 * We can ignore bit 0.
230 */
231 *error = status & (RTE_IOAT_CHANSTS_SUSPENDED | RTE_IOAT_CHANSTS_ARMED);
232 return (status - ioat->ring_addr) >> 6;
233 }
234
235 /*
236 * Returns details of operations that have been completed
237 */
238 static __rte_always_inline int
__ioat_completed_ops(int dev_id,uint8_t max_copies,uintptr_t * src_hdls,uintptr_t * dst_hdls)239 __ioat_completed_ops(int dev_id, uint8_t max_copies,
240 uintptr_t *src_hdls, uintptr_t *dst_hdls)
241 {
242 struct rte_ioat_rawdev *ioat =
243 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
244 unsigned short mask = (ioat->ring_size - 1);
245 unsigned short read = ioat->next_read;
246 unsigned short end_read, count;
247 int error;
248 int i = 0;
249
250 end_read = (__ioat_get_last_completed(ioat, &error) + 1) & mask;
251 count = (end_read - (read & mask)) & mask;
252
253 if (error) {
254 rte_errno = EIO;
255 return -1;
256 }
257
258 if (ioat->hdls_disable) {
259 read += count;
260 goto end;
261 }
262
263 if (count > max_copies)
264 count = max_copies;
265
266 for (; i < count - 1; i += 2, read += 2) {
267 __m128i hdls0 = _mm_load_si128(&ioat->hdls[read & mask]);
268 __m128i hdls1 = _mm_load_si128(&ioat->hdls[(read + 1) & mask]);
269
270 _mm_storeu_si128((__m128i *)&src_hdls[i],
271 _mm_unpacklo_epi64(hdls0, hdls1));
272 _mm_storeu_si128((__m128i *)&dst_hdls[i],
273 _mm_unpackhi_epi64(hdls0, hdls1));
274 }
275 for (; i < count; i++, read++) {
276 uintptr_t *hdls = (uintptr_t *)&ioat->hdls[read & mask];
277 src_hdls[i] = hdls[0];
278 dst_hdls[i] = hdls[1];
279 }
280
281 end:
282 ioat->next_read = read;
283 ioat->xstats.completed += count;
284 return count;
285 }
286
287 static inline uint16_t
rte_ioat_burst_capacity(int dev_id)288 rte_ioat_burst_capacity(int dev_id)
289 {
290 enum rte_ioat_dev_type *type =
291 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
292 if (*type == RTE_IDXD_DEV)
293 return __idxd_burst_capacity(dev_id);
294 else
295 return __ioat_burst_capacity(dev_id);
296 }
297
298 static inline int
rte_ioat_enqueue_fill(int dev_id,uint64_t pattern,phys_addr_t dst,unsigned int len,uintptr_t dst_hdl)299 rte_ioat_enqueue_fill(int dev_id, uint64_t pattern, phys_addr_t dst,
300 unsigned int len, uintptr_t dst_hdl)
301 {
302 enum rte_ioat_dev_type *type =
303 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
304 if (*type == RTE_IDXD_DEV)
305 return __idxd_enqueue_fill(dev_id, pattern, dst, len, dst_hdl);
306 else
307 return __ioat_enqueue_fill(dev_id, pattern, dst, len, dst_hdl);
308 }
309
310 static inline int
rte_ioat_enqueue_copy(int dev_id,phys_addr_t src,phys_addr_t dst,unsigned int length,uintptr_t src_hdl,uintptr_t dst_hdl)311 rte_ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst,
312 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
313 {
314 enum rte_ioat_dev_type *type =
315 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
316 if (*type == RTE_IDXD_DEV)
317 return __idxd_enqueue_copy(dev_id, src, dst, length,
318 src_hdl, dst_hdl);
319 else
320 return __ioat_enqueue_copy(dev_id, src, dst, length,
321 src_hdl, dst_hdl);
322 }
323
324 static inline int
rte_ioat_fence(int dev_id)325 rte_ioat_fence(int dev_id)
326 {
327 enum rte_ioat_dev_type *type =
328 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
329 if (*type == RTE_IDXD_DEV)
330 return __idxd_fence(dev_id);
331 else
332 return __ioat_fence(dev_id);
333 }
334
335 static inline int
rte_ioat_perform_ops(int dev_id)336 rte_ioat_perform_ops(int dev_id)
337 {
338 enum rte_ioat_dev_type *type =
339 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
340 if (*type == RTE_IDXD_DEV)
341 return __idxd_perform_ops(dev_id);
342 else
343 return __ioat_perform_ops(dev_id);
344 }
345
346 static inline int
rte_ioat_completed_ops(int dev_id,uint8_t max_copies,uint32_t * status,uint8_t * num_unsuccessful,uintptr_t * src_hdls,uintptr_t * dst_hdls)347 rte_ioat_completed_ops(int dev_id, uint8_t max_copies,
348 uint32_t *status, uint8_t *num_unsuccessful,
349 uintptr_t *src_hdls, uintptr_t *dst_hdls)
350 {
351 enum rte_ioat_dev_type *type =
352 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
353 uint8_t tmp; /* used so functions don't need to check for null parameter */
354
355 if (num_unsuccessful == NULL)
356 num_unsuccessful = &tmp;
357
358 *num_unsuccessful = 0;
359 if (*type == RTE_IDXD_DEV)
360 return __idxd_completed_ops(dev_id, max_copies, status, num_unsuccessful,
361 src_hdls, dst_hdls);
362 else
363 return __ioat_completed_ops(dev_id, max_copies, src_hdls, dst_hdls);
364 }
365
366 static inline void
367 __rte_deprecated_msg("use rte_ioat_perform_ops() instead")
rte_ioat_do_copies(int dev_id)368 rte_ioat_do_copies(int dev_id) { rte_ioat_perform_ops(dev_id); }
369
370 static inline int
371 __rte_deprecated_msg("use rte_ioat_completed_ops() instead")
rte_ioat_completed_copies(int dev_id,uint8_t max_copies,uintptr_t * src_hdls,uintptr_t * dst_hdls)372 rte_ioat_completed_copies(int dev_id, uint8_t max_copies,
373 uintptr_t *src_hdls, uintptr_t *dst_hdls)
374 {
375 return rte_ioat_completed_ops(dev_id, max_copies, NULL, NULL,
376 src_hdls, dst_hdls);
377 }
378
379 #endif /* _RTE_IOAT_RAWDEV_FNS_H_ */
380