1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2020 Intel Corporation
3 */
4
5 #include <sys/uio.h>
6 #ifdef RTE_RAW_IOAT
7 #include <rte_rawdev.h>
8 #include <rte_ioat_rawdev.h>
9
10 #include "ioat.h"
11 #include "main.h"
12
13 struct dma_for_vhost dma_bind[MAX_VHOST_DEVICE];
14
15 struct packet_tracker {
16 unsigned short size_track[MAX_ENQUEUED_SIZE];
17 unsigned short next_read;
18 unsigned short next_write;
19 unsigned short last_remain;
20 };
21
22 struct packet_tracker cb_tracker[MAX_VHOST_DEVICE];
23
24
25 int
open_ioat(const char * value)26 open_ioat(const char *value)
27 {
28 struct dma_for_vhost *dma_info = dma_bind;
29 char *input = strndup(value, strlen(value) + 1);
30 char *addrs = input;
31 char *ptrs[2];
32 char *start, *end, *substr;
33 int64_t vid, vring_id;
34 struct rte_ioat_rawdev_config config;
35 struct rte_rawdev_info info = { .dev_private = &config };
36 char name[32];
37 int dev_id;
38 int ret = 0;
39 uint16_t i = 0;
40 char *dma_arg[MAX_VHOST_DEVICE];
41 int args_nr;
42
43 while (isblank(*addrs))
44 addrs++;
45 if (*addrs == '\0') {
46 ret = -1;
47 goto out;
48 }
49
50 /* process DMA devices within bracket. */
51 addrs++;
52 substr = strtok(addrs, ";]");
53 if (!substr) {
54 ret = -1;
55 goto out;
56 }
57 args_nr = rte_strsplit(substr, strlen(substr),
58 dma_arg, MAX_VHOST_DEVICE, ',');
59 if (args_nr <= 0) {
60 ret = -1;
61 goto out;
62 }
63 while (i < args_nr) {
64 char *arg_temp = dma_arg[i];
65 uint8_t sub_nr;
66 sub_nr = rte_strsplit(arg_temp, strlen(arg_temp), ptrs, 2, '@');
67 if (sub_nr != 2) {
68 ret = -1;
69 goto out;
70 }
71
72 start = strstr(ptrs[0], "txd");
73 if (start == NULL) {
74 ret = -1;
75 goto out;
76 }
77
78 start += 3;
79 vid = strtol(start, &end, 0);
80 if (end == start) {
81 ret = -1;
82 goto out;
83 }
84
85 vring_id = 0 + VIRTIO_RXQ;
86 if (rte_pci_addr_parse(ptrs[1],
87 &(dma_info + vid)->dmas[vring_id].addr) < 0) {
88 ret = -1;
89 goto out;
90 }
91
92 rte_pci_device_name(&(dma_info + vid)->dmas[vring_id].addr,
93 name, sizeof(name));
94 dev_id = rte_rawdev_get_dev_id(name);
95 if (dev_id == (uint16_t)(-ENODEV) ||
96 dev_id == (uint16_t)(-EINVAL)) {
97 ret = -1;
98 goto out;
99 }
100
101 if (rte_rawdev_info_get(dev_id, &info, sizeof(config)) < 0 ||
102 strstr(info.driver_name, "ioat") == NULL) {
103 ret = -1;
104 goto out;
105 }
106
107 (dma_info + vid)->dmas[vring_id].dev_id = dev_id;
108 (dma_info + vid)->dmas[vring_id].is_valid = true;
109 config.ring_size = IOAT_RING_SIZE;
110 config.hdls_disable = true;
111 if (rte_rawdev_configure(dev_id, &info, sizeof(config)) < 0) {
112 ret = -1;
113 goto out;
114 }
115 rte_rawdev_start(dev_id);
116
117 dma_info->nr++;
118 i++;
119 }
120 out:
121 free(input);
122 return ret;
123 }
124
125 uint32_t
ioat_transfer_data_cb(int vid,uint16_t queue_id,struct rte_vhost_async_desc * descs,struct rte_vhost_async_status * opaque_data,uint16_t count)126 ioat_transfer_data_cb(int vid, uint16_t queue_id,
127 struct rte_vhost_async_desc *descs,
128 struct rte_vhost_async_status *opaque_data, uint16_t count)
129 {
130 uint32_t i_desc;
131 int dev_id = dma_bind[vid].dmas[queue_id * 2 + VIRTIO_RXQ].dev_id;
132 struct rte_vhost_iov_iter *src = NULL;
133 struct rte_vhost_iov_iter *dst = NULL;
134 unsigned long i_seg;
135 unsigned short mask = MAX_ENQUEUED_SIZE - 1;
136 unsigned short write = cb_tracker[dev_id].next_write;
137
138 if (!opaque_data) {
139 for (i_desc = 0; i_desc < count; i_desc++) {
140 src = descs[i_desc].src;
141 dst = descs[i_desc].dst;
142 i_seg = 0;
143 while (i_seg < src->nr_segs) {
144 /*
145 * TODO: Assuming that the ring space of the
146 * IOAT device is large enough, so there is no
147 * error here, and the actual error handling
148 * will be added later.
149 */
150 rte_ioat_enqueue_copy(dev_id,
151 (uintptr_t)(src->iov[i_seg].iov_base)
152 + src->offset,
153 (uintptr_t)(dst->iov[i_seg].iov_base)
154 + dst->offset,
155 src->iov[i_seg].iov_len,
156 0,
157 0);
158 i_seg++;
159 }
160 write &= mask;
161 cb_tracker[dev_id].size_track[write] = i_seg;
162 write++;
163 }
164 } else {
165 /* Opaque data is not supported */
166 return -1;
167 }
168 /* ring the doorbell */
169 rte_ioat_perform_ops(dev_id);
170 cb_tracker[dev_id].next_write = write;
171 return i_desc;
172 }
173
174 uint32_t
ioat_check_completed_copies_cb(int vid,uint16_t queue_id,struct rte_vhost_async_status * opaque_data,uint16_t max_packets)175 ioat_check_completed_copies_cb(int vid, uint16_t queue_id,
176 struct rte_vhost_async_status *opaque_data,
177 uint16_t max_packets)
178 {
179 if (!opaque_data) {
180 uintptr_t dump[255];
181 unsigned short n_seg;
182 unsigned short read, write;
183 unsigned short nb_packet = 0;
184 unsigned short mask = MAX_ENQUEUED_SIZE - 1;
185 unsigned short i;
186 int dev_id = dma_bind[vid].dmas[queue_id * 2
187 + VIRTIO_RXQ].dev_id;
188 n_seg = rte_ioat_completed_ops(dev_id, 255, dump, dump);
189 n_seg += cb_tracker[dev_id].last_remain;
190 if (!n_seg)
191 return 0;
192 read = cb_tracker[dev_id].next_read;
193 write = cb_tracker[dev_id].next_write;
194 for (i = 0; i < max_packets; i++) {
195 read &= mask;
196 if (read == write)
197 break;
198 if (n_seg >= cb_tracker[dev_id].size_track[read]) {
199 n_seg -= cb_tracker[dev_id].size_track[read];
200 read++;
201 nb_packet++;
202 } else {
203 break;
204 }
205 }
206 cb_tracker[dev_id].next_read = read;
207 cb_tracker[dev_id].last_remain = n_seg;
208 return nb_packet;
209 }
210 /* Opaque data is not supported */
211 return -1;
212 }
213
214 #endif /* RTE_RAW_IOAT */
215