1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2020 Intel Corporation
3 */
4
5 #include <stdint.h>
6 #include <stdbool.h>
7 #include <stdio.h>
8 #include <errno.h>
9 #include <assert.h>
10 #include <unistd.h>
11 #include <string.h>
12
13 #include <rte_malloc.h>
14 #include <rte_errno.h>
15
16 #include "base/dlb_resource.h"
17 #include "base/dlb_osdep.h"
18 #include "base/dlb_regs.h"
19 #include "../dlb_priv.h"
20 #include "../dlb_inline_fns.h"
21 #include "../dlb_user.h"
22 #include "dlb_main.h"
23
24 unsigned int dlb_unregister_timeout_s = DLB_DEFAULT_UNREGISTER_TIMEOUT_S;
25
26 #define DLB_PCI_CAP_POINTER 0x34
27 #define DLB_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
28 #define DLB_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
29 #define DLB_PCI_ERR_UNCOR_MASK 8
30 #define DLB_PCI_ERR_UNC_UNSUP 0x00100000
31
32 #define DLB_PCI_LNKCTL 16
33 #define DLB_PCI_SLTCTL 24
34 #define DLB_PCI_RTCTL 28
35 #define DLB_PCI_EXP_DEVCTL2 40
36 #define DLB_PCI_LNKCTL2 48
37 #define DLB_PCI_SLTCTL2 56
38 #define DLB_PCI_CMD 4
39 #define DLB_PCI_EXP_DEVSTA 10
40 #define DLB_PCI_EXP_DEVSTA_TRPND 0x20
41 #define DLB_PCI_EXP_DEVCTL_BCR_FLR 0x8000
42
43 #define DLB_PCI_CAP_ID_EXP 0x10
44 #define DLB_PCI_CAP_ID_MSIX 0x11
45 #define DLB_PCI_EXT_CAP_ID_PRI 0x13
46 #define DLB_PCI_EXT_CAP_ID_ACS 0xD
47
48 #define DLB_PCI_PRI_CTRL_ENABLE 0x1
49 #define DLB_PCI_PRI_ALLOC_REQ 0xC
50 #define DLB_PCI_PRI_CTRL 0x4
51 #define DLB_PCI_MSIX_FLAGS 0x2
52 #define DLB_PCI_MSIX_FLAGS_ENABLE 0x8000
53 #define DLB_PCI_MSIX_FLAGS_MASKALL 0x4000
54 #define DLB_PCI_ERR_ROOT_STATUS 0x30
55 #define DLB_PCI_ERR_COR_STATUS 0x10
56 #define DLB_PCI_ERR_UNCOR_STATUS 0x4
57 #define DLB_PCI_COMMAND_INTX_DISABLE 0x400
58 #define DLB_PCI_ACS_CAP 0x4
59 #define DLB_PCI_ACS_CTRL 0x6
60 #define DLB_PCI_ACS_SV 0x1
61 #define DLB_PCI_ACS_RR 0x4
62 #define DLB_PCI_ACS_CR 0x8
63 #define DLB_PCI_ACS_UF 0x10
64 #define DLB_PCI_ACS_EC 0x20
65
dlb_pci_find_capability(struct rte_pci_device * pdev,uint32_t id)66 static int dlb_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
67 {
68 uint8_t pos;
69 int ret;
70 uint16_t hdr;
71
72 ret = rte_pci_read_config(pdev, &pos, 1, DLB_PCI_CAP_POINTER);
73 pos &= 0xFC;
74
75 if (ret != 1)
76 return -1;
77
78 while (pos > 0x3F) {
79 ret = rte_pci_read_config(pdev, &hdr, 2, pos);
80 if (ret != 2)
81 return -1;
82
83 if (DLB_PCI_CAP_ID(hdr) == id)
84 return pos;
85
86 if (DLB_PCI_CAP_ID(hdr) == 0xFF)
87 return -1;
88
89 pos = DLB_PCI_CAP_NEXT(hdr);
90 }
91
92 return -1;
93 }
94
dlb_mask_ur_err(struct rte_pci_device * pdev)95 static int dlb_mask_ur_err(struct rte_pci_device *pdev)
96 {
97 uint32_t mask;
98 size_t sz = sizeof(mask);
99 int pos = rte_pci_find_ext_capability(pdev, RTE_PCI_EXT_CAP_ID_ERR);
100
101 if (pos < 0) {
102 DLB_LOG_ERR("[%s()] failed to find the aer capability\n",
103 __func__);
104 return pos;
105 }
106
107 pos += DLB_PCI_ERR_UNCOR_MASK;
108
109 if (rte_pci_read_config(pdev, &mask, sz, pos) != (int)sz) {
110 DLB_LOG_ERR("[%s()] Failed to read uncorrectable error mask reg\n",
111 __func__);
112 return -1;
113 }
114
115 /* Mask Unsupported Request errors */
116 mask |= DLB_PCI_ERR_UNC_UNSUP;
117
118 if (rte_pci_write_config(pdev, &mask, sz, pos) != (int)sz) {
119 DLB_LOG_ERR("[%s()] Failed to write uncorrectable error mask reg at offset %d\n",
120 __func__, pos);
121 return -1;
122 }
123
124 return 0;
125 }
126
127 struct dlb_dev *
dlb_probe(struct rte_pci_device * pdev)128 dlb_probe(struct rte_pci_device *pdev)
129 {
130 struct dlb_dev *dlb_dev;
131 int ret = 0;
132
133 DLB_INFO(dlb_dev, "probe\n");
134
135 dlb_dev = rte_malloc("DLB_PF", sizeof(struct dlb_dev),
136 RTE_CACHE_LINE_SIZE);
137
138 if (dlb_dev == NULL) {
139 ret = -ENOMEM;
140 goto dlb_dev_malloc_fail;
141 }
142
143 /* PCI Bus driver has already mapped bar space into process.
144 * Save off our IO register and FUNC addresses.
145 */
146
147 /* BAR 0 */
148 if (pdev->mem_resource[0].addr == NULL) {
149 DLB_ERR(dlb_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
150 ret = -EINVAL;
151 goto pci_mmap_bad_addr;
152 }
153 dlb_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
154 dlb_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
155
156 DLB_INFO(dlb_dev, "DLB FUNC VA=%p, PA=%p, len=%"PRIu64"\n",
157 (void *)dlb_dev->hw.func_kva,
158 (void *)dlb_dev->hw.func_phys_addr,
159 pdev->mem_resource[0].len);
160
161 /* BAR 2 */
162 if (pdev->mem_resource[2].addr == NULL) {
163 DLB_ERR(dlb_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
164 ret = -EINVAL;
165 goto pci_mmap_bad_addr;
166 }
167 dlb_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
168 dlb_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
169
170 DLB_INFO(dlb_dev, "DLB CSR VA=%p, PA=%p, len=%"PRIu64"\n",
171 (void *)dlb_dev->hw.csr_kva,
172 (void *)dlb_dev->hw.csr_phys_addr,
173 pdev->mem_resource[2].len);
174
175 dlb_dev->pdev = pdev;
176
177 ret = dlb_pf_reset(dlb_dev);
178 if (ret)
179 goto dlb_reset_fail;
180
181 /* DLB incorrectly sends URs in response to certain messages. Mask UR
182 * errors to prevent these from being propagated to the MCA.
183 */
184 ret = dlb_mask_ur_err(pdev);
185 if (ret)
186 goto mask_ur_err_fail;
187
188 ret = dlb_pf_init_driver_state(dlb_dev);
189 if (ret)
190 goto init_driver_state_fail;
191
192 ret = dlb_resource_init(&dlb_dev->hw);
193 if (ret)
194 goto resource_init_fail;
195
196 dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
197
198 dlb_pf_init_hardware(dlb_dev);
199
200 return dlb_dev;
201
202 resource_init_fail:
203 dlb_resource_free(&dlb_dev->hw);
204 init_driver_state_fail:
205 mask_ur_err_fail:
206 dlb_reset_fail:
207 pci_mmap_bad_addr:
208 rte_free(dlb_dev);
209 dlb_dev_malloc_fail:
210 rte_errno = ret;
211 return NULL;
212 }
213
214 int
dlb_pf_reset(struct dlb_dev * dlb_dev)215 dlb_pf_reset(struct dlb_dev *dlb_dev)
216 {
217 int msix_cap_offset, err_cap_offset, acs_cap_offset, wait_count;
218 uint16_t dev_ctl_word, dev_ctl2_word, lnk_word, lnk_word2;
219 uint16_t rt_ctl_word, pri_ctrl_word;
220 struct rte_pci_device *pdev = dlb_dev->pdev;
221 uint16_t devsta_busy_word, devctl_word;
222 int pcie_cap_offset, pri_cap_offset;
223 uint16_t slt_word, slt_word2, cmd;
224 int ret = 0, i = 0;
225 uint32_t dword[16], pri_reqs_dword;
226 off_t off;
227
228 /* Save PCI config state */
229
230 for (i = 0; i < 16; i++) {
231 if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
232 return ret;
233 }
234
235 pcie_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_EXP);
236
237 if (pcie_cap_offset < 0) {
238 DLB_LOG_ERR("[%s()] failed to find the pcie capability\n",
239 __func__);
240 return pcie_cap_offset;
241 }
242
243 off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL;
244 if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
245 dev_ctl_word = 0;
246
247 off = pcie_cap_offset + DLB_PCI_LNKCTL;
248 if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
249 lnk_word = 0;
250
251 off = pcie_cap_offset + DLB_PCI_SLTCTL;
252 if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
253 slt_word = 0;
254
255 off = pcie_cap_offset + DLB_PCI_RTCTL;
256 if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
257 rt_ctl_word = 0;
258
259 off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
260 if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
261 dev_ctl2_word = 0;
262
263 off = pcie_cap_offset + DLB_PCI_LNKCTL2;
264 if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
265 lnk_word2 = 0;
266
267 off = pcie_cap_offset + DLB_PCI_SLTCTL2;
268 if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
269 slt_word2 = 0;
270
271 pri_cap_offset = rte_pci_find_ext_capability(pdev,
272 DLB_PCI_EXT_CAP_ID_PRI);
273 if (pri_cap_offset >= 0) {
274 off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
275 if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
276 pri_reqs_dword = 0;
277 }
278
279 /* clear the PCI command register before issuing the FLR */
280
281 off = DLB_PCI_CMD;
282 cmd = 0;
283 if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
284 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
285 __func__, (int)off);
286 return -1;
287 }
288
289 /* issue the FLR */
290 for (wait_count = 0; wait_count < 4; wait_count++) {
291 int sleep_time;
292
293 off = pcie_cap_offset + DLB_PCI_EXP_DEVSTA;
294 ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
295 if (ret != 2) {
296 DLB_LOG_ERR("[%s()] failed to read the pci device status\n",
297 __func__);
298 return ret;
299 }
300
301 if (!(devsta_busy_word & DLB_PCI_EXP_DEVSTA_TRPND))
302 break;
303
304 sleep_time = (1 << (wait_count)) * 100;
305 rte_delay_ms(sleep_time);
306 }
307
308 if (wait_count == 4) {
309 DLB_LOG_ERR("[%s()] wait for pci pending transactions timed out\n",
310 __func__);
311 return -1;
312 }
313
314 off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL;
315 ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
316 if (ret != 2) {
317 DLB_LOG_ERR("[%s()] failed to read the pcie device control\n",
318 __func__);
319 return ret;
320 }
321
322 devctl_word |= DLB_PCI_EXP_DEVCTL_BCR_FLR;
323
324 if (rte_pci_write_config(pdev, &devctl_word, 2, off) != 2) {
325 DLB_LOG_ERR("[%s()] failed to write the pcie device control at offset %d\n",
326 __func__, (int)off);
327 return -1;
328 }
329
330 rte_delay_ms(100);
331
332 /* Restore PCI config state */
333
334 if (pcie_cap_offset >= 0) {
335 off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL;
336 if (rte_pci_write_config(pdev, &dev_ctl_word, 2, off) != 2) {
337 DLB_LOG_ERR("[%s()] failed to write the pcie device control at offset %d\n",
338 __func__, (int)off);
339 return -1;
340 }
341
342 off = pcie_cap_offset + DLB_PCI_LNKCTL;
343 if (rte_pci_write_config(pdev, &lnk_word, 2, off) != 2) {
344 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
345 __func__, (int)off);
346 return -1;
347 }
348
349 off = pcie_cap_offset + DLB_PCI_SLTCTL;
350 if (rte_pci_write_config(pdev, &slt_word, 2, off) != 2) {
351 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
352 __func__, (int)off);
353 return -1;
354 }
355
356 off = pcie_cap_offset + DLB_PCI_RTCTL;
357 if (rte_pci_write_config(pdev, &rt_ctl_word, 2, off) != 2) {
358 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
359 __func__, (int)off);
360 return -1;
361 }
362
363 off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
364 if (rte_pci_write_config(pdev, &dev_ctl2_word, 2, off) != 2) {
365 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
366 __func__, (int)off);
367 return -1;
368 }
369
370 off = pcie_cap_offset + DLB_PCI_LNKCTL2;
371 if (rte_pci_write_config(pdev, &lnk_word2, 2, off) != 2) {
372 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
373 __func__, (int)off);
374 return -1;
375 }
376
377 off = pcie_cap_offset + DLB_PCI_SLTCTL2;
378 if (rte_pci_write_config(pdev, &slt_word2, 2, off) != 2) {
379 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
380 __func__, (int)off);
381 return -1;
382 }
383 }
384
385 if (pri_cap_offset >= 0) {
386 pri_ctrl_word = DLB_PCI_PRI_CTRL_ENABLE;
387
388 off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
389 if (rte_pci_write_config(pdev, &pri_reqs_dword, 4, off) != 4) {
390 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
391 __func__, (int)off);
392 return -1;
393 }
394
395 off = pri_cap_offset + DLB_PCI_PRI_CTRL;
396 if (rte_pci_write_config(pdev, &pri_ctrl_word, 2, off) != 2) {
397 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
398 __func__, (int)off);
399 return -1;
400 }
401 }
402
403 err_cap_offset = rte_pci_find_ext_capability(pdev,
404 RTE_PCI_EXT_CAP_ID_ERR);
405 if (err_cap_offset >= 0) {
406 uint32_t tmp;
407
408 off = err_cap_offset + DLB_PCI_ERR_ROOT_STATUS;
409 if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
410 tmp = 0;
411
412 if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
413 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
414 __func__, (int)off);
415 return -1;
416 }
417
418 off = err_cap_offset + DLB_PCI_ERR_COR_STATUS;
419 if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
420 tmp = 0;
421
422 if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
423 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
424 __func__, (int)off);
425 return -1;
426 }
427
428 off = err_cap_offset + DLB_PCI_ERR_UNCOR_STATUS;
429 if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
430 tmp = 0;
431
432 if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
433 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
434 __func__, (int)off);
435 return -1;
436 }
437 }
438
439 for (i = 16; i > 0; i--) {
440 off = (i - 1) * 4;
441 if (rte_pci_write_config(pdev, &dword[i - 1], 4, off) != 4) {
442 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
443 __func__, (int)off);
444 return -1;
445 }
446 }
447
448 off = DLB_PCI_CMD;
449 if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
450 cmd &= ~DLB_PCI_COMMAND_INTX_DISABLE;
451 if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
452 DLB_LOG_ERR("[%s()] failed to write pci config space\n",
453 __func__);
454 return -1;
455 }
456 }
457
458 msix_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_MSIX);
459 if (msix_cap_offset >= 0) {
460 off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
461 if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
462 cmd |= DLB_PCI_MSIX_FLAGS_ENABLE;
463 cmd |= DLB_PCI_MSIX_FLAGS_MASKALL;
464 if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
465 DLB_LOG_ERR("[%s()] failed to write msix flags\n",
466 __func__);
467 return -1;
468 }
469 }
470
471 off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
472 if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
473 cmd &= ~DLB_PCI_MSIX_FLAGS_MASKALL;
474 if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
475 DLB_LOG_ERR("[%s()] failed to write msix flags\n",
476 __func__);
477 return -1;
478 }
479 }
480 }
481
482 acs_cap_offset = rte_pci_find_ext_capability(pdev,
483 DLB_PCI_EXT_CAP_ID_ACS);
484 if (acs_cap_offset >= 0) {
485 uint16_t acs_cap, acs_ctrl, acs_mask;
486 off = acs_cap_offset + DLB_PCI_ACS_CAP;
487 if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
488 acs_cap = 0;
489
490 off = acs_cap_offset + DLB_PCI_ACS_CTRL;
491 if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
492 acs_ctrl = 0;
493
494 acs_mask = DLB_PCI_ACS_SV | DLB_PCI_ACS_RR;
495 acs_mask |= (DLB_PCI_ACS_CR | DLB_PCI_ACS_UF);
496 acs_ctrl |= (acs_cap & acs_mask);
497
498 if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
499 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
500 __func__, (int)off);
501 return -1;
502 }
503
504 off = acs_cap_offset + DLB_PCI_ACS_CTRL;
505 if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
506 acs_ctrl = 0;
507
508 acs_mask = DLB_PCI_ACS_RR | DLB_PCI_ACS_CR | DLB_PCI_ACS_EC;
509 acs_ctrl &= ~acs_mask;
510
511 off = acs_cap_offset + DLB_PCI_ACS_CTRL;
512 if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
513 DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
514 __func__, (int)off);
515 return -1;
516 }
517 }
518
519 return 0;
520 }
521
522 /*******************************/
523 /****** Driver management ******/
524 /*******************************/
525
526 int
dlb_pf_init_driver_state(struct dlb_dev * dlb_dev)527 dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
528 {
529 /* Initialize software state */
530 rte_spinlock_init(&dlb_dev->resource_mutex);
531 rte_spinlock_init(&dlb_dev->measurement_lock);
532
533 return 0;
534 }
535
536 void
dlb_pf_init_hardware(struct dlb_dev * dlb_dev)537 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
538 {
539 dlb_disable_dp_vasr_feature(&dlb_dev->hw);
540
541 dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
542
543 if (dlb_dev->revision >= DLB_REV_B0) {
544 dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
545 dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
546 }
547
548 if (dlb_dev->revision >= DLB_REV_B0) {
549 dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
550 dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
551 }
552 }
553