1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018-2020 Intel Corporation
3 */
4
5 #include <rte_string_fns.h>
6 #include <rte_devargs.h>
7 #include <ctype.h>
8
9 #include "qat_device.h"
10 #include "adf_transport_access_macros.h"
11 #include "qat_sym_pmd.h"
12 #include "qat_comp_pmd.h"
13
14 /* Hardware device information per generation */
15 __extension__
16 struct qat_gen_hw_data qat_gen_config[] = {
17 [QAT_GEN1] = {
18 .dev_gen = QAT_GEN1,
19 .qp_hw_data = qat_gen1_qps,
20 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN1
21 },
22 [QAT_GEN2] = {
23 .dev_gen = QAT_GEN2,
24 .qp_hw_data = qat_gen1_qps,
25 /* gen2 has same ring layout as gen1 */
26 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN2
27 },
28 [QAT_GEN3] = {
29 .dev_gen = QAT_GEN3,
30 .qp_hw_data = qat_gen3_qps,
31 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
32 },
33 };
34
35 /* per-process array of device data */
36 struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES];
37 static int qat_nb_pci_devices;
38
39 /*
40 * The set of PCI devices this driver supports
41 */
42
43 static const struct rte_pci_id pci_id_qat_map[] = {
44 {
45 RTE_PCI_DEVICE(0x8086, 0x0443),
46 },
47 {
48 RTE_PCI_DEVICE(0x8086, 0x37c9),
49 },
50 {
51 RTE_PCI_DEVICE(0x8086, 0x19e3),
52 },
53 {
54 RTE_PCI_DEVICE(0x8086, 0x6f55),
55 },
56 {
57 RTE_PCI_DEVICE(0x8086, 0x18ef),
58 },
59 {
60 RTE_PCI_DEVICE(0x8086, 0x18a1),
61 },
62 {.device_id = 0},
63 };
64
65 static struct qat_pci_device *
qat_pci_get_named_dev(const char * name)66 qat_pci_get_named_dev(const char *name)
67 {
68 unsigned int i;
69
70 if (name == NULL)
71 return NULL;
72
73 for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) {
74 if (qat_pci_devs[i].mz &&
75 (strcmp(((struct qat_pci_device *)
76 qat_pci_devs[i].mz->addr)->name, name)
77 == 0))
78 return (struct qat_pci_device *)
79 qat_pci_devs[i].mz->addr;
80 }
81
82 return NULL;
83 }
84
85 static uint8_t
qat_pci_find_free_device_index(void)86 qat_pci_find_free_device_index(void)
87 {
88 uint8_t dev_id;
89
90 for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES;
91 dev_id++) {
92 if (qat_pci_devs[dev_id].mz == NULL)
93 break;
94 }
95 return dev_id;
96 }
97
98 struct qat_pci_device *
qat_get_qat_dev_from_pci_dev(struct rte_pci_device * pci_dev)99 qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev)
100 {
101 char name[QAT_DEV_NAME_MAX_LEN];
102
103 rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
104
105 return qat_pci_get_named_dev(name);
106 }
107
qat_dev_parse_cmd(const char * str,struct qat_dev_cmd_param * qat_dev_cmd_param)108 static void qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param
109 *qat_dev_cmd_param)
110 {
111 int i = 0;
112 const char *param;
113
114 while (1) {
115 char value_str[4] = { };
116
117 param = qat_dev_cmd_param[i].name;
118 if (param == NULL)
119 return;
120 long value = 0;
121 const char *arg = strstr(str, param);
122 const char *arg2 = NULL;
123
124 if (arg) {
125 arg2 = arg + strlen(param);
126 if (*arg2 != '=') {
127 QAT_LOG(DEBUG, "parsing error '=' sign"
128 " should immediately follow %s",
129 param);
130 arg2 = NULL;
131 } else
132 arg2++;
133 } else {
134 QAT_LOG(DEBUG, "%s not provided", param);
135 }
136 if (arg2) {
137 int iter = 0;
138 while (iter < 2) {
139 if (!isdigit(*(arg2 + iter)))
140 break;
141 iter++;
142 }
143 if (!iter) {
144 QAT_LOG(DEBUG, "parsing error %s"
145 " no number provided",
146 param);
147 } else {
148 memcpy(value_str, arg2, iter);
149 value = strtol(value_str, NULL, 10);
150 if (value > MAX_QP_THRESHOLD_SIZE) {
151 QAT_LOG(DEBUG, "Exceeded max size of"
152 " threshold, setting to %d",
153 MAX_QP_THRESHOLD_SIZE);
154 value = MAX_QP_THRESHOLD_SIZE;
155 }
156 QAT_LOG(DEBUG, "parsing %s = %ld",
157 param, value);
158 }
159 }
160 qat_dev_cmd_param[i].val = value;
161 i++;
162 }
163 }
164
165 struct qat_pci_device *
qat_pci_device_allocate(struct rte_pci_device * pci_dev,struct qat_dev_cmd_param * qat_dev_cmd_param)166 qat_pci_device_allocate(struct rte_pci_device *pci_dev,
167 struct qat_dev_cmd_param *qat_dev_cmd_param)
168 {
169 struct qat_pci_device *qat_dev;
170 uint8_t qat_dev_id = 0;
171 char name[QAT_DEV_NAME_MAX_LEN];
172 struct rte_devargs *devargs = pci_dev->device.devargs;
173
174 rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
175 snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
176
177 if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
178 const struct rte_memzone *mz = rte_memzone_lookup(name);
179
180 if (mz == NULL) {
181 QAT_LOG(ERR,
182 "Secondary can't find %s mz, did primary create device?",
183 name);
184 return NULL;
185 }
186 qat_dev = mz->addr;
187 qat_pci_devs[qat_dev->qat_dev_id].mz = mz;
188 qat_pci_devs[qat_dev->qat_dev_id].pci_dev = pci_dev;
189 qat_nb_pci_devices++;
190 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
191 qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
192 return qat_dev;
193 }
194
195 if (qat_pci_get_named_dev(name) != NULL) {
196 QAT_LOG(ERR, "QAT device with name %s already allocated!",
197 name);
198 return NULL;
199 }
200
201 qat_dev_id = qat_pci_find_free_device_index();
202 if (qat_dev_id == RTE_PMD_QAT_MAX_PCI_DEVICES) {
203 QAT_LOG(ERR, "Reached maximum number of QAT devices");
204 return NULL;
205 }
206
207 qat_pci_devs[qat_dev_id].mz = rte_memzone_reserve(name,
208 sizeof(struct qat_pci_device),
209 rte_socket_id(), 0);
210
211 if (qat_pci_devs[qat_dev_id].mz == NULL) {
212 QAT_LOG(ERR, "Error when allocating memzone for QAT_%d",
213 qat_dev_id);
214 return NULL;
215 }
216
217 qat_dev = qat_pci_devs[qat_dev_id].mz->addr;
218 memset(qat_dev, 0, sizeof(*qat_dev));
219 strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN);
220 qat_dev->qat_dev_id = qat_dev_id;
221 qat_pci_devs[qat_dev_id].pci_dev = pci_dev;
222 switch (pci_dev->id.device_id) {
223 case 0x0443:
224 qat_dev->qat_dev_gen = QAT_GEN1;
225 break;
226 case 0x37c9:
227 case 0x19e3:
228 case 0x6f55:
229 case 0x18ef:
230 qat_dev->qat_dev_gen = QAT_GEN2;
231 break;
232 case 0x18a1:
233 qat_dev->qat_dev_gen = QAT_GEN3;
234 break;
235 default:
236 QAT_LOG(ERR, "Invalid dev_id, can't determine generation");
237 rte_memzone_free(qat_pci_devs[qat_dev->qat_dev_id].mz);
238 return NULL;
239 }
240
241 if (devargs && devargs->drv_str)
242 qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param);
243
244 rte_spinlock_init(&qat_dev->arb_csr_lock);
245 qat_nb_pci_devices++;
246
247 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
248 qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
249
250 return qat_dev;
251 }
252
253 static int
qat_pci_device_release(struct rte_pci_device * pci_dev)254 qat_pci_device_release(struct rte_pci_device *pci_dev)
255 {
256 struct qat_pci_device *qat_dev;
257 char name[QAT_DEV_NAME_MAX_LEN];
258 int busy = 0;
259
260 if (pci_dev == NULL)
261 return -EINVAL;
262
263 rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
264 snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
265 qat_dev = qat_pci_get_named_dev(name);
266 if (qat_dev != NULL) {
267
268 struct qat_device_info *inst =
269 &qat_pci_devs[qat_dev->qat_dev_id];
270 /* Check that there are no service devs still on pci device */
271
272 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
273 if (qat_dev->sym_dev != NULL) {
274 QAT_LOG(DEBUG, "QAT sym device %s is busy",
275 name);
276 busy = 1;
277 }
278 if (qat_dev->asym_dev != NULL) {
279 QAT_LOG(DEBUG, "QAT asym device %s is busy",
280 name);
281 busy = 1;
282 }
283 if (qat_dev->comp_dev != NULL) {
284 QAT_LOG(DEBUG, "QAT comp device %s is busy",
285 name);
286 busy = 1;
287 }
288 if (busy)
289 return -EBUSY;
290 rte_memzone_free(inst->mz);
291 }
292 memset(inst, 0, sizeof(struct qat_device_info));
293 qat_nb_pci_devices--;
294 QAT_LOG(DEBUG, "QAT device %s released, total QATs %d",
295 name, qat_nb_pci_devices);
296 }
297 return 0;
298 }
299
300 static int
qat_pci_dev_destroy(struct qat_pci_device * qat_pci_dev,struct rte_pci_device * pci_dev)301 qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev,
302 struct rte_pci_device *pci_dev)
303 {
304 qat_sym_dev_destroy(qat_pci_dev);
305 qat_comp_dev_destroy(qat_pci_dev);
306 qat_asym_dev_destroy(qat_pci_dev);
307 return qat_pci_device_release(pci_dev);
308 }
309
qat_pci_probe(struct rte_pci_driver * pci_drv __rte_unused,struct rte_pci_device * pci_dev)310 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
311 struct rte_pci_device *pci_dev)
312 {
313 int sym_ret = 0, asym_ret = 0, comp_ret = 0;
314 int num_pmds_created = 0;
315 struct qat_pci_device *qat_pci_dev;
316 struct qat_dev_cmd_param qat_dev_cmd_param[] = {
317 { SYM_ENQ_THRESHOLD_NAME, 0 },
318 { ASYM_ENQ_THRESHOLD_NAME, 0 },
319 { COMP_ENQ_THRESHOLD_NAME, 0 },
320 { NULL, 0 },
321 };
322
323 QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
324 pci_dev->addr.bus,
325 pci_dev->addr.devid,
326 pci_dev->addr.function);
327
328 qat_pci_dev = qat_pci_device_allocate(pci_dev, qat_dev_cmd_param);
329 if (qat_pci_dev == NULL)
330 return -ENODEV;
331
332 sym_ret = qat_sym_dev_create(qat_pci_dev, qat_dev_cmd_param);
333 if (sym_ret == 0) {
334 num_pmds_created++;
335
336 }
337 else
338 QAT_LOG(WARNING,
339 "Failed to create QAT SYM PMD on device %s",
340 qat_pci_dev->name);
341
342 comp_ret = qat_comp_dev_create(qat_pci_dev, qat_dev_cmd_param);
343 if (comp_ret == 0)
344 num_pmds_created++;
345 else
346 QAT_LOG(WARNING,
347 "Failed to create QAT COMP PMD on device %s",
348 qat_pci_dev->name);
349
350 asym_ret = qat_asym_dev_create(qat_pci_dev, qat_dev_cmd_param);
351 if (asym_ret == 0)
352 num_pmds_created++;
353 else
354 QAT_LOG(WARNING,
355 "Failed to create QAT ASYM PMD on device %s",
356 qat_pci_dev->name);
357
358 if (num_pmds_created == 0)
359 qat_pci_dev_destroy(qat_pci_dev, pci_dev);
360
361 return 0;
362 }
363
qat_pci_remove(struct rte_pci_device * pci_dev)364 static int qat_pci_remove(struct rte_pci_device *pci_dev)
365 {
366 struct qat_pci_device *qat_pci_dev;
367
368 if (pci_dev == NULL)
369 return -EINVAL;
370
371 qat_pci_dev = qat_get_qat_dev_from_pci_dev(pci_dev);
372 if (qat_pci_dev == NULL)
373 return 0;
374
375 return qat_pci_dev_destroy(qat_pci_dev, pci_dev);
376 }
377
378 static struct rte_pci_driver rte_qat_pmd = {
379 .id_table = pci_id_qat_map,
380 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
381 .probe = qat_pci_probe,
382 .remove = qat_pci_remove
383 };
384
385 __rte_weak int
qat_sym_dev_create(struct qat_pci_device * qat_pci_dev __rte_unused,struct qat_dev_cmd_param * qat_dev_cmd_param __rte_unused)386 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
387 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
388 {
389 return 0;
390 }
391
392 __rte_weak int
qat_asym_dev_create(struct qat_pci_device * qat_pci_dev __rte_unused,struct qat_dev_cmd_param * qat_dev_cmd_param __rte_unused)393 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
394 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
395 {
396 return 0;
397 }
398
399 __rte_weak int
qat_sym_dev_destroy(struct qat_pci_device * qat_pci_dev __rte_unused)400 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
401 {
402 return 0;
403 }
404
405 __rte_weak int
qat_asym_dev_destroy(struct qat_pci_device * qat_pci_dev __rte_unused)406 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
407 {
408 return 0;
409 }
410
411 __rte_weak int
qat_comp_dev_create(struct qat_pci_device * qat_pci_dev __rte_unused,struct qat_dev_cmd_param * qat_dev_cmd_param __rte_unused)412 qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
413 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
414 {
415 return 0;
416 }
417
418 __rte_weak int
qat_comp_dev_destroy(struct qat_pci_device * qat_pci_dev __rte_unused)419 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
420 {
421 return 0;
422 }
423
424 RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd);
425 RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map);
426 RTE_PMD_REGISTER_KMOD_DEP(QAT_PCI_NAME, "* igb_uio | uio_pci_generic | vfio-pci");
427