1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
4 * Copyright 2016-2019 NXP
5 *
6 */
7
8 #include <unistd.h>
9 #include <stdio.h>
10 #include <sys/types.h>
11 #include <string.h>
12 #include <stdlib.h>
13 #include <fcntl.h>
14 #include <errno.h>
15 #include <sys/ioctl.h>
16 #include <sys/stat.h>
17 #include <sys/mman.h>
18 #include <sys/vfs.h>
19 #include <libgen.h>
20 #include <dirent.h>
21 #include <sys/eventfd.h>
22
23 #include <eal_filesystem.h>
24 #include <rte_mbuf.h>
25 #include <rte_ethdev_driver.h>
26 #include <rte_malloc.h>
27 #include <rte_memcpy.h>
28 #include <rte_string_fns.h>
29 #include <rte_cycles.h>
30 #include <rte_kvargs.h>
31 #include <rte_dev.h>
32 #include <rte_bus.h>
33 #include <rte_eal_memconfig.h>
34
35 #include "rte_fslmc.h"
36 #include "fslmc_vfio.h"
37 #include "fslmc_logs.h"
38 #include <mc/fsl_dpmng.h>
39
40 #include "portal/dpaa2_hw_pvt.h"
41 #include "portal/dpaa2_hw_dpio.h"
42
43 #define FSLMC_CONTAINER_MAX_LEN 8 /**< Of the format dprc.XX */
44
45 /* Number of VFIO containers & groups with in */
46 static struct fslmc_vfio_group vfio_group;
47 static struct fslmc_vfio_container vfio_container;
48 static int container_device_fd;
49 char *fslmc_container;
50 static int fslmc_iommu_type;
51 static uint32_t *msi_intr_vaddr;
52 void *(*rte_mcp_ptr_list);
53
54 void *
dpaa2_get_mcp_ptr(int portal_idx)55 dpaa2_get_mcp_ptr(int portal_idx)
56 {
57 if (rte_mcp_ptr_list)
58 return rte_mcp_ptr_list[portal_idx];
59 else
60 return NULL;
61 }
62
63 static struct rte_dpaa2_object_list dpaa2_obj_list =
64 TAILQ_HEAD_INITIALIZER(dpaa2_obj_list);
65
66 /*register a fslmc bus based dpaa2 driver */
67 void
rte_fslmc_object_register(struct rte_dpaa2_object * object)68 rte_fslmc_object_register(struct rte_dpaa2_object *object)
69 {
70 RTE_VERIFY(object);
71
72 TAILQ_INSERT_TAIL(&dpaa2_obj_list, object, next);
73 }
74
75 int
fslmc_get_container_group(int * groupid)76 fslmc_get_container_group(int *groupid)
77 {
78 int ret;
79 char *container;
80
81 if (!fslmc_container) {
82 container = getenv("DPRC");
83 if (container == NULL) {
84 DPAA2_BUS_DEBUG("DPAA2: DPRC not available");
85 return -EINVAL;
86 }
87
88 if (strlen(container) >= FSLMC_CONTAINER_MAX_LEN) {
89 DPAA2_BUS_ERR("Invalid container name: %s", container);
90 return -1;
91 }
92
93 fslmc_container = strdup(container);
94 if (!fslmc_container) {
95 DPAA2_BUS_ERR("Mem alloc failure; Container name");
96 return -ENOMEM;
97 }
98 }
99
100 fslmc_iommu_type = (rte_vfio_noiommu_is_enabled() == 1) ?
101 RTE_VFIO_NOIOMMU : VFIO_TYPE1_IOMMU;
102
103 /* get group number */
104 ret = rte_vfio_get_group_num(SYSFS_FSL_MC_DEVICES,
105 fslmc_container, groupid);
106 if (ret <= 0) {
107 DPAA2_BUS_ERR("Unable to find %s IOMMU group", fslmc_container);
108 return -1;
109 }
110
111 DPAA2_BUS_DEBUG("Container: %s has VFIO iommu group id = %d",
112 fslmc_container, *groupid);
113
114 return 0;
115 }
116
117 static int
vfio_connect_container(void)118 vfio_connect_container(void)
119 {
120 int fd, ret;
121
122 if (vfio_container.used) {
123 DPAA2_BUS_DEBUG("No container available");
124 return -1;
125 }
126
127 /* Try connecting to vfio container if already created */
128 if (!ioctl(vfio_group.fd, VFIO_GROUP_SET_CONTAINER,
129 &vfio_container.fd)) {
130 DPAA2_BUS_DEBUG(
131 "Container pre-exists with FD[0x%x] for this group",
132 vfio_container.fd);
133 vfio_group.container = &vfio_container;
134 return 0;
135 }
136
137 /* Opens main vfio file descriptor which represents the "container" */
138 fd = rte_vfio_get_container_fd();
139 if (fd < 0) {
140 DPAA2_BUS_ERR("Failed to open VFIO container");
141 return -errno;
142 }
143
144 /* Check whether support for SMMU type IOMMU present or not */
145 if (ioctl(fd, VFIO_CHECK_EXTENSION, fslmc_iommu_type)) {
146 /* Connect group to container */
147 ret = ioctl(vfio_group.fd, VFIO_GROUP_SET_CONTAINER, &fd);
148 if (ret) {
149 DPAA2_BUS_ERR("Failed to setup group container");
150 close(fd);
151 return -errno;
152 }
153
154 ret = ioctl(fd, VFIO_SET_IOMMU, fslmc_iommu_type);
155 if (ret) {
156 DPAA2_BUS_ERR("Failed to setup VFIO iommu");
157 close(fd);
158 return -errno;
159 }
160 } else {
161 DPAA2_BUS_ERR("No supported IOMMU available");
162 close(fd);
163 return -EINVAL;
164 }
165
166 vfio_container.used = 1;
167 vfio_container.fd = fd;
168 vfio_container.group = &vfio_group;
169 vfio_group.container = &vfio_container;
170
171 return 0;
172 }
173
vfio_map_irq_region(struct fslmc_vfio_group * group)174 static int vfio_map_irq_region(struct fslmc_vfio_group *group)
175 {
176 int ret;
177 unsigned long *vaddr = NULL;
178 struct vfio_iommu_type1_dma_map map = {
179 .argsz = sizeof(map),
180 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
181 .vaddr = 0x6030000,
182 .iova = 0x6030000,
183 .size = 0x1000,
184 };
185
186 vaddr = (unsigned long *)mmap(NULL, 0x1000, PROT_WRITE |
187 PROT_READ, MAP_SHARED, container_device_fd, 0x6030000);
188 if (vaddr == MAP_FAILED) {
189 DPAA2_BUS_INFO("Unable to map region (errno = %d)", errno);
190 return -errno;
191 }
192
193 msi_intr_vaddr = (uint32_t *)((char *)(vaddr) + 64);
194 map.vaddr = (unsigned long)vaddr;
195 ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA, &map);
196 if (ret == 0)
197 return 0;
198
199 DPAA2_BUS_ERR("Unable to map DMA address (errno = %d)", errno);
200 return -errno;
201 }
202
203 static int fslmc_map_dma(uint64_t vaddr, rte_iova_t iovaddr, size_t len);
204 static int fslmc_unmap_dma(uint64_t vaddr, rte_iova_t iovaddr, size_t len);
205
206 static void
fslmc_memevent_cb(enum rte_mem_event type,const void * addr,size_t len,void * arg __rte_unused)207 fslmc_memevent_cb(enum rte_mem_event type, const void *addr, size_t len,
208 void *arg __rte_unused)
209 {
210 struct rte_memseg_list *msl;
211 struct rte_memseg *ms;
212 size_t cur_len = 0, map_len = 0;
213 uint64_t virt_addr;
214 rte_iova_t iova_addr;
215 int ret;
216
217 msl = rte_mem_virt2memseg_list(addr);
218
219 while (cur_len < len) {
220 const void *va = RTE_PTR_ADD(addr, cur_len);
221
222 ms = rte_mem_virt2memseg(va, msl);
223 iova_addr = ms->iova;
224 virt_addr = ms->addr_64;
225 map_len = ms->len;
226
227 DPAA2_BUS_DEBUG("Request for %s, va=%p, "
228 "virt_addr=0x%" PRIx64 ", "
229 "iova=0x%" PRIx64 ", map_len=%zu",
230 type == RTE_MEM_EVENT_ALLOC ?
231 "alloc" : "dealloc",
232 va, virt_addr, iova_addr, map_len);
233
234 /* iova_addr may be set to RTE_BAD_IOVA */
235 if (iova_addr == RTE_BAD_IOVA) {
236 DPAA2_BUS_DEBUG("Segment has invalid iova, skipping\n");
237 cur_len += map_len;
238 continue;
239 }
240
241 if (type == RTE_MEM_EVENT_ALLOC)
242 ret = fslmc_map_dma(virt_addr, iova_addr, map_len);
243 else
244 ret = fslmc_unmap_dma(virt_addr, iova_addr, map_len);
245
246 if (ret != 0) {
247 DPAA2_BUS_ERR("DMA Mapping/Unmapping failed. "
248 "Map=%d, addr=%p, len=%zu, err:(%d)",
249 type, va, map_len, ret);
250 return;
251 }
252
253 cur_len += map_len;
254 }
255
256 if (type == RTE_MEM_EVENT_ALLOC)
257 DPAA2_BUS_DEBUG("Total Mapped: addr=%p, len=%zu",
258 addr, len);
259 else
260 DPAA2_BUS_DEBUG("Total Unmapped: addr=%p, len=%zu",
261 addr, len);
262 }
263
264 static int
fslmc_map_dma(uint64_t vaddr,rte_iova_t iovaddr __rte_unused,size_t len)265 fslmc_map_dma(uint64_t vaddr, rte_iova_t iovaddr __rte_unused, size_t len)
266 {
267 struct fslmc_vfio_group *group;
268 struct vfio_iommu_type1_dma_map dma_map = {
269 .argsz = sizeof(struct vfio_iommu_type1_dma_map),
270 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
271 };
272 int ret;
273
274 if (fslmc_iommu_type == RTE_VFIO_NOIOMMU) {
275 DPAA2_BUS_DEBUG("Running in NOIOMMU mode");
276 return 0;
277 }
278
279 dma_map.size = len;
280 dma_map.vaddr = vaddr;
281
282 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
283 dma_map.iova = iovaddr;
284 #else
285 dma_map.iova = dma_map.vaddr;
286 #endif
287
288 /* SET DMA MAP for IOMMU */
289 group = &vfio_group;
290
291 if (!group->container) {
292 DPAA2_BUS_ERR("Container is not connected ");
293 return -1;
294 }
295
296 DPAA2_BUS_DEBUG("--> Map address: 0x%"PRIx64", size: %"PRIu64"",
297 (uint64_t)dma_map.vaddr, (uint64_t)dma_map.size);
298 ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA, &dma_map);
299 if (ret) {
300 DPAA2_BUS_ERR("VFIO_IOMMU_MAP_DMA API(errno = %d)",
301 errno);
302 return -1;
303 }
304
305 return 0;
306 }
307
308 static int
fslmc_unmap_dma(uint64_t vaddr,uint64_t iovaddr __rte_unused,size_t len)309 fslmc_unmap_dma(uint64_t vaddr, uint64_t iovaddr __rte_unused, size_t len)
310 {
311 struct fslmc_vfio_group *group;
312 struct vfio_iommu_type1_dma_unmap dma_unmap = {
313 .argsz = sizeof(struct vfio_iommu_type1_dma_unmap),
314 .flags = 0,
315 };
316 int ret;
317
318 if (fslmc_iommu_type == RTE_VFIO_NOIOMMU) {
319 DPAA2_BUS_DEBUG("Running in NOIOMMU mode");
320 return 0;
321 }
322
323 dma_unmap.size = len;
324 dma_unmap.iova = vaddr;
325
326 /* SET DMA MAP for IOMMU */
327 group = &vfio_group;
328
329 if (!group->container) {
330 DPAA2_BUS_ERR("Container is not connected ");
331 return -1;
332 }
333
334 DPAA2_BUS_DEBUG("--> Unmap address: 0x%"PRIx64", size: %"PRIu64"",
335 (uint64_t)dma_unmap.iova, (uint64_t)dma_unmap.size);
336 ret = ioctl(group->container->fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap);
337 if (ret) {
338 DPAA2_BUS_ERR("VFIO_IOMMU_UNMAP_DMA API(errno = %d)",
339 errno);
340 return -1;
341 }
342
343 return 0;
344 }
345
346 static int
fslmc_dmamap_seg(const struct rte_memseg_list * msl __rte_unused,const struct rte_memseg * ms,void * arg)347 fslmc_dmamap_seg(const struct rte_memseg_list *msl __rte_unused,
348 const struct rte_memseg *ms, void *arg)
349 {
350 int *n_segs = arg;
351 int ret;
352
353 /* if IOVA address is invalid, skip */
354 if (ms->iova == RTE_BAD_IOVA)
355 return 0;
356
357 ret = fslmc_map_dma(ms->addr_64, ms->iova, ms->len);
358 if (ret)
359 DPAA2_BUS_ERR("Unable to VFIO map (addr=%p, len=%zu)",
360 ms->addr, ms->len);
361 else
362 (*n_segs)++;
363
364 return ret;
365 }
366
367 int
rte_fslmc_vfio_mem_dmamap(uint64_t vaddr,uint64_t iova,uint64_t size)368 rte_fslmc_vfio_mem_dmamap(uint64_t vaddr, uint64_t iova, uint64_t size)
369 {
370 int ret;
371 struct fslmc_vfio_group *group;
372 struct vfio_iommu_type1_dma_map dma_map = {
373 .argsz = sizeof(struct vfio_iommu_type1_dma_map),
374 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
375 };
376
377 if (fslmc_iommu_type == RTE_VFIO_NOIOMMU) {
378 DPAA2_BUS_DEBUG("Running in NOIOMMU mode");
379 return 0;
380 }
381
382 /* SET DMA MAP for IOMMU */
383 group = &vfio_group;
384 if (!group->container) {
385 DPAA2_BUS_ERR("Container is not connected");
386 return -1;
387 }
388
389 dma_map.size = size;
390 dma_map.vaddr = vaddr;
391 dma_map.iova = iova;
392
393 DPAA2_BUS_DEBUG("VFIOdmamap 0x%"PRIx64":0x%"PRIx64",size 0x%"PRIx64"\n",
394 (uint64_t)dma_map.vaddr, (uint64_t)dma_map.iova,
395 (uint64_t)dma_map.size);
396 ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA,
397 &dma_map);
398 if (ret) {
399 printf("Unable to map DMA address (errno = %d)\n",
400 errno);
401 return ret;
402 }
403
404 return 0;
405 }
406
rte_fslmc_vfio_dmamap(void)407 int rte_fslmc_vfio_dmamap(void)
408 {
409 int i = 0, ret;
410
411 /* Lock before parsing and registering callback to memory subsystem */
412 rte_mcfg_mem_read_lock();
413
414 if (rte_memseg_walk(fslmc_dmamap_seg, &i) < 0) {
415 rte_mcfg_mem_read_unlock();
416 return -1;
417 }
418
419 ret = rte_mem_event_callback_register("fslmc_memevent_clb",
420 fslmc_memevent_cb, NULL);
421 if (ret && rte_errno == ENOTSUP)
422 DPAA2_BUS_DEBUG("Memory event callbacks not supported");
423 else if (ret)
424 DPAA2_BUS_DEBUG("Unable to install memory handler");
425 else
426 DPAA2_BUS_DEBUG("Installed memory callback handler");
427
428 DPAA2_BUS_DEBUG("Total %d segments found.", i);
429
430 /* TODO - This is a W.A. as VFIO currently does not add the mapping of
431 * the interrupt region to SMMU. This should be removed once the
432 * support is added in the Kernel.
433 */
434 vfio_map_irq_region(&vfio_group);
435
436 /* Existing segments have been mapped and memory callback for hotplug
437 * has been installed.
438 */
439 rte_mcfg_mem_read_unlock();
440
441 return 0;
442 }
443
444 static int
fslmc_vfio_setup_device(const char * sysfs_base,const char * dev_addr,int * vfio_dev_fd,struct vfio_device_info * device_info)445 fslmc_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
446 int *vfio_dev_fd, struct vfio_device_info *device_info)
447 {
448 struct vfio_group_status group_status = {
449 .argsz = sizeof(group_status)
450 };
451 int vfio_group_fd, vfio_container_fd, iommu_group_no, ret;
452
453 /* get group number */
454 ret = rte_vfio_get_group_num(sysfs_base, dev_addr, &iommu_group_no);
455 if (ret < 0)
456 return -1;
457
458 /* get the actual group fd */
459 vfio_group_fd = rte_vfio_get_group_fd(iommu_group_no);
460 if (vfio_group_fd < 0 && vfio_group_fd != -ENOENT)
461 return -1;
462
463 /*
464 * if vfio_group_fd == -ENOENT, that means the device
465 * isn't managed by VFIO
466 */
467 if (vfio_group_fd == -ENOENT) {
468 RTE_LOG(WARNING, EAL, " %s not managed by VFIO driver, skipping\n",
469 dev_addr);
470 return 1;
471 }
472
473 /* Opens main vfio file descriptor which represents the "container" */
474 vfio_container_fd = rte_vfio_get_container_fd();
475 if (vfio_container_fd < 0) {
476 DPAA2_BUS_ERR("Failed to open VFIO container");
477 return -errno;
478 }
479
480 /* check if the group is viable */
481 ret = ioctl(vfio_group_fd, VFIO_GROUP_GET_STATUS, &group_status);
482 if (ret) {
483 DPAA2_BUS_ERR(" %s cannot get group status, "
484 "error %i (%s)\n", dev_addr,
485 errno, strerror(errno));
486 close(vfio_group_fd);
487 rte_vfio_clear_group(vfio_group_fd);
488 return -1;
489 } else if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
490 DPAA2_BUS_ERR(" %s VFIO group is not viable!\n", dev_addr);
491 close(vfio_group_fd);
492 rte_vfio_clear_group(vfio_group_fd);
493 return -1;
494 }
495 /* At this point, we know that this group is viable (meaning,
496 * all devices are either bound to VFIO or not bound to anything)
497 */
498
499 /* check if group does not have a container yet */
500 if (!(group_status.flags & VFIO_GROUP_FLAGS_CONTAINER_SET)) {
501
502 /* add group to a container */
503 ret = ioctl(vfio_group_fd, VFIO_GROUP_SET_CONTAINER,
504 &vfio_container_fd);
505 if (ret) {
506 DPAA2_BUS_ERR(" %s cannot add VFIO group to container, "
507 "error %i (%s)\n", dev_addr,
508 errno, strerror(errno));
509 close(vfio_group_fd);
510 close(vfio_container_fd);
511 rte_vfio_clear_group(vfio_group_fd);
512 return -1;
513 }
514
515 /*
516 * set an IOMMU type for container
517 *
518 */
519 if (ioctl(vfio_container_fd, VFIO_CHECK_EXTENSION,
520 fslmc_iommu_type)) {
521 ret = ioctl(vfio_container_fd, VFIO_SET_IOMMU,
522 fslmc_iommu_type);
523 if (ret) {
524 DPAA2_BUS_ERR("Failed to setup VFIO iommu");
525 close(vfio_group_fd);
526 close(vfio_container_fd);
527 return -errno;
528 }
529 } else {
530 DPAA2_BUS_ERR("No supported IOMMU available");
531 close(vfio_group_fd);
532 close(vfio_container_fd);
533 return -EINVAL;
534 }
535 }
536
537 /* get a file descriptor for the device */
538 *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
539 if (*vfio_dev_fd < 0) {
540 /* if we cannot get a device fd, this implies a problem with
541 * the VFIO group or the container not having IOMMU configured.
542 */
543
544 DPAA2_BUS_WARN("Getting a vfio_dev_fd for %s failed", dev_addr);
545 close(vfio_group_fd);
546 close(vfio_container_fd);
547 rte_vfio_clear_group(vfio_group_fd);
548 return -1;
549 }
550
551 /* test and setup the device */
552 ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
553 if (ret) {
554 DPAA2_BUS_ERR(" %s cannot get device info, error %i (%s)",
555 dev_addr, errno, strerror(errno));
556 close(*vfio_dev_fd);
557 close(vfio_group_fd);
558 close(vfio_container_fd);
559 rte_vfio_clear_group(vfio_group_fd);
560 return -1;
561 }
562
563 return 0;
564 }
565
vfio_map_mcp_obj(const char * mcp_obj)566 static intptr_t vfio_map_mcp_obj(const char *mcp_obj)
567 {
568 intptr_t v_addr = (intptr_t)MAP_FAILED;
569 int32_t ret, mc_fd;
570 struct vfio_group_status status = { .argsz = sizeof(status) };
571
572 struct vfio_device_info d_info = { .argsz = sizeof(d_info) };
573 struct vfio_region_info reg_info = { .argsz = sizeof(reg_info) };
574
575 fslmc_vfio_setup_device(SYSFS_FSL_MC_DEVICES, mcp_obj,
576 &mc_fd, &d_info);
577
578 /* getting device region info*/
579 ret = ioctl(mc_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info);
580 if (ret < 0) {
581 DPAA2_BUS_ERR("Error in VFIO getting REGION_INFO");
582 goto MC_FAILURE;
583 }
584
585 v_addr = (size_t)mmap(NULL, reg_info.size,
586 PROT_WRITE | PROT_READ, MAP_SHARED,
587 mc_fd, reg_info.offset);
588
589 MC_FAILURE:
590 close(mc_fd);
591
592 return v_addr;
593 }
594
595 #define IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + sizeof(int))
596
rte_dpaa2_intr_enable(struct rte_intr_handle * intr_handle,int index)597 int rte_dpaa2_intr_enable(struct rte_intr_handle *intr_handle, int index)
598 {
599 int len, ret;
600 char irq_set_buf[IRQ_SET_BUF_LEN];
601 struct vfio_irq_set *irq_set;
602 int *fd_ptr;
603
604 len = sizeof(irq_set_buf);
605
606 irq_set = (struct vfio_irq_set *)irq_set_buf;
607 irq_set->argsz = len;
608 irq_set->count = 1;
609 irq_set->flags =
610 VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
611 irq_set->index = index;
612 irq_set->start = 0;
613 fd_ptr = (int *)&irq_set->data;
614 *fd_ptr = intr_handle->fd;
615
616 ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
617 if (ret) {
618 DPAA2_BUS_ERR("Error:dpaa2 SET IRQs fd=%d, err = %d(%s)",
619 intr_handle->fd, errno, strerror(errno));
620 return ret;
621 }
622
623 return ret;
624 }
625
rte_dpaa2_intr_disable(struct rte_intr_handle * intr_handle,int index)626 int rte_dpaa2_intr_disable(struct rte_intr_handle *intr_handle, int index)
627 {
628 struct vfio_irq_set *irq_set;
629 char irq_set_buf[IRQ_SET_BUF_LEN];
630 int len, ret;
631
632 len = sizeof(struct vfio_irq_set);
633
634 irq_set = (struct vfio_irq_set *)irq_set_buf;
635 irq_set->argsz = len;
636 irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
637 irq_set->index = index;
638 irq_set->start = 0;
639 irq_set->count = 0;
640
641 ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
642 if (ret)
643 DPAA2_BUS_ERR(
644 "Error disabling dpaa2 interrupts for fd %d",
645 intr_handle->fd);
646
647 return ret;
648 }
649
650 /* set up interrupt support (but not enable interrupts) */
651 int
rte_dpaa2_vfio_setup_intr(struct rte_intr_handle * intr_handle,int vfio_dev_fd,int num_irqs)652 rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
653 int vfio_dev_fd,
654 int num_irqs)
655 {
656 int i, ret;
657
658 /* start from MSI-X interrupt type */
659 for (i = 0; i < num_irqs; i++) {
660 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
661 int fd = -1;
662
663 irq_info.index = i;
664
665 ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
666 if (ret < 0) {
667 DPAA2_BUS_ERR("Cannot get IRQ(%d) info, error %i (%s)",
668 i, errno, strerror(errno));
669 return -1;
670 }
671
672 /* if this vector cannot be used with eventfd,
673 * fail if we explicitly
674 * specified interrupt type, otherwise continue
675 */
676 if ((irq_info.flags & VFIO_IRQ_INFO_EVENTFD) == 0)
677 continue;
678
679 /* set up an eventfd for interrupts */
680 fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
681 if (fd < 0) {
682 DPAA2_BUS_ERR("Cannot set up eventfd, error %i (%s)",
683 errno, strerror(errno));
684 return -1;
685 }
686
687 intr_handle->fd = fd;
688 intr_handle->type = RTE_INTR_HANDLE_VFIO_MSI;
689 intr_handle->vfio_dev_fd = vfio_dev_fd;
690
691 return 0;
692 }
693
694 /* if we're here, we haven't found a suitable interrupt vector */
695 return -1;
696 }
697
698 /*
699 * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly
700 * EVENT) devices.
701 */
702 static int
fslmc_process_iodevices(struct rte_dpaa2_device * dev)703 fslmc_process_iodevices(struct rte_dpaa2_device *dev)
704 {
705 int dev_fd;
706 struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
707 struct rte_dpaa2_object *object = NULL;
708
709 fslmc_vfio_setup_device(SYSFS_FSL_MC_DEVICES, dev->device.name,
710 &dev_fd, &device_info);
711
712 switch (dev->dev_type) {
713 case DPAA2_ETH:
714 rte_dpaa2_vfio_setup_intr(&dev->intr_handle, dev_fd,
715 device_info.num_irqs);
716 break;
717 case DPAA2_CON:
718 case DPAA2_IO:
719 case DPAA2_CI:
720 case DPAA2_BPOOL:
721 case DPAA2_DPRTC:
722 case DPAA2_MUX:
723 TAILQ_FOREACH(object, &dpaa2_obj_list, next) {
724 if (dev->dev_type == object->dev_type)
725 object->create(dev_fd, &device_info,
726 dev->object_id);
727 else
728 continue;
729 }
730 break;
731 default:
732 break;
733 }
734
735 DPAA2_BUS_LOG(DEBUG, "Device (%s) abstracted from VFIO",
736 dev->device.name);
737 return 0;
738 }
739
740 static int
fslmc_process_mcp(struct rte_dpaa2_device * dev)741 fslmc_process_mcp(struct rte_dpaa2_device *dev)
742 {
743 int ret;
744 intptr_t v_addr;
745 struct fsl_mc_io dpmng = {0};
746 struct mc_version mc_ver_info = {0};
747
748 rte_mcp_ptr_list = malloc(sizeof(void *) * (MC_PORTAL_INDEX + 1));
749 if (!rte_mcp_ptr_list) {
750 DPAA2_BUS_ERR("Unable to allocate MC portal memory");
751 ret = -ENOMEM;
752 goto cleanup;
753 }
754
755 v_addr = vfio_map_mcp_obj(dev->device.name);
756 if (v_addr == (intptr_t)MAP_FAILED) {
757 DPAA2_BUS_ERR("Error mapping region (errno = %d)", errno);
758 ret = -1;
759 goto cleanup;
760 }
761
762 /* check the MC version compatibility */
763 dpmng.regs = (void *)v_addr;
764
765 /* In case of secondary processes, MC version check is no longer
766 * required.
767 */
768 if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
769 rte_mcp_ptr_list[MC_PORTAL_INDEX] = (void *)v_addr;
770 return 0;
771 }
772
773 if (mc_get_version(&dpmng, CMD_PRI_LOW, &mc_ver_info)) {
774 DPAA2_BUS_ERR("Unable to obtain MC version");
775 ret = -1;
776 goto cleanup;
777 }
778
779 if ((mc_ver_info.major != MC_VER_MAJOR) ||
780 (mc_ver_info.minor < MC_VER_MINOR)) {
781 DPAA2_BUS_ERR("DPAA2 MC version not compatible!"
782 " Expected %d.%d.x, Detected %d.%d.%d",
783 MC_VER_MAJOR, MC_VER_MINOR,
784 mc_ver_info.major, mc_ver_info.minor,
785 mc_ver_info.revision);
786 ret = -1;
787 goto cleanup;
788 }
789 rte_mcp_ptr_list[MC_PORTAL_INDEX] = (void *)v_addr;
790
791 return 0;
792
793 cleanup:
794 if (rte_mcp_ptr_list) {
795 free(rte_mcp_ptr_list);
796 rte_mcp_ptr_list = NULL;
797 }
798
799 return ret;
800 }
801
802 int
fslmc_vfio_process_group(void)803 fslmc_vfio_process_group(void)
804 {
805 int ret;
806 int found_mportal = 0;
807 struct rte_dpaa2_device *dev, *dev_temp;
808 bool is_dpmcp_in_blocklist = false, is_dpio_in_blocklist = false;
809 int dpmcp_count = 0, dpio_count = 0, current_device;
810
811 TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
812 if (dev->dev_type == DPAA2_MPORTAL) {
813 dpmcp_count++;
814 if (dev->device.devargs &&
815 dev->device.devargs->policy == RTE_DEV_BLOCKED)
816 is_dpmcp_in_blocklist = true;
817 }
818 if (dev->dev_type == DPAA2_IO) {
819 dpio_count++;
820 if (dev->device.devargs &&
821 dev->device.devargs->policy == RTE_DEV_BLOCKED)
822 is_dpio_in_blocklist = true;
823 }
824 }
825
826 /* Search the MCP as that should be initialized first. */
827 current_device = 0;
828 TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
829 if (dev->dev_type == DPAA2_MPORTAL) {
830 current_device++;
831 if (dev->device.devargs &&
832 dev->device.devargs->policy == RTE_DEV_BLOCKED) {
833 DPAA2_BUS_LOG(DEBUG, "%s Blocked, skipping",
834 dev->device.name);
835 TAILQ_REMOVE(&rte_fslmc_bus.device_list,
836 dev, next);
837 continue;
838 }
839
840 if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
841 !is_dpmcp_in_blocklist) {
842 if (dpmcp_count == 1 ||
843 current_device != dpmcp_count) {
844 TAILQ_REMOVE(&rte_fslmc_bus.device_list,
845 dev, next);
846 continue;
847 }
848 }
849
850 if (!found_mportal) {
851 ret = fslmc_process_mcp(dev);
852 if (ret) {
853 DPAA2_BUS_ERR("Unable to map MC Portal");
854 return -1;
855 }
856 found_mportal = 1;
857 }
858
859 TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
860 free(dev);
861 dev = NULL;
862 /* Ideally there is only a single dpmcp, but in case
863 * multiple exists, looping on remaining devices.
864 */
865 }
866 }
867
868 /* Cannot continue if there is not even a single mportal */
869 if (!found_mportal) {
870 DPAA2_BUS_ERR("No MC Portal device found. Not continuing");
871 return -1;
872 }
873
874 current_device = 0;
875 TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
876 if (dev->dev_type == DPAA2_IO)
877 current_device++;
878 if (dev->device.devargs &&
879 dev->device.devargs->policy == RTE_DEV_BLOCKED) {
880 DPAA2_BUS_LOG(DEBUG, "%s Blocked, skipping",
881 dev->device.name);
882 TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
883 continue;
884 }
885 if (rte_eal_process_type() == RTE_PROC_SECONDARY &&
886 dev->dev_type != DPAA2_ETH &&
887 dev->dev_type != DPAA2_CRYPTO &&
888 dev->dev_type != DPAA2_QDMA &&
889 dev->dev_type != DPAA2_IO) {
890 TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
891 continue;
892 }
893 switch (dev->dev_type) {
894 case DPAA2_ETH:
895 case DPAA2_CRYPTO:
896 case DPAA2_QDMA:
897 ret = fslmc_process_iodevices(dev);
898 if (ret) {
899 DPAA2_BUS_DEBUG("Dev (%s) init failed",
900 dev->device.name);
901 return ret;
902 }
903 break;
904 case DPAA2_CON:
905 case DPAA2_CI:
906 case DPAA2_BPOOL:
907 case DPAA2_DPRTC:
908 case DPAA2_MUX:
909 /* IN case of secondary processes, all control objects
910 * like dpbp, dpcon, dpci are not initialized/required
911 * - all of these are assumed to be initialized and made
912 * available by primary.
913 */
914 if (rte_eal_process_type() == RTE_PROC_SECONDARY)
915 continue;
916
917 /* Call the object creation routine and remove the
918 * device entry from device list
919 */
920 ret = fslmc_process_iodevices(dev);
921 if (ret) {
922 DPAA2_BUS_DEBUG("Dev (%s) init failed",
923 dev->device.name);
924 return -1;
925 }
926
927 break;
928 case DPAA2_IO:
929 if (!is_dpio_in_blocklist && dpio_count > 1) {
930 if (rte_eal_process_type() == RTE_PROC_SECONDARY
931 && current_device != dpio_count) {
932 TAILQ_REMOVE(&rte_fslmc_bus.device_list,
933 dev, next);
934 break;
935 }
936 if (rte_eal_process_type() == RTE_PROC_PRIMARY
937 && current_device == dpio_count) {
938 TAILQ_REMOVE(&rte_fslmc_bus.device_list,
939 dev, next);
940 break;
941 }
942 }
943
944 ret = fslmc_process_iodevices(dev);
945 if (ret) {
946 DPAA2_BUS_DEBUG("Dev (%s) init failed",
947 dev->device.name);
948 return -1;
949 }
950
951 break;
952 case DPAA2_UNKNOWN:
953 default:
954 /* Unknown - ignore */
955 DPAA2_BUS_DEBUG("Found unknown device (%s)",
956 dev->device.name);
957 TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
958 free(dev);
959 dev = NULL;
960 }
961 }
962
963 return 0;
964 }
965
966 int
fslmc_vfio_setup_group(void)967 fslmc_vfio_setup_group(void)
968 {
969 int groupid;
970 int ret;
971 struct vfio_group_status status = { .argsz = sizeof(status) };
972
973 /* if already done once */
974 if (container_device_fd)
975 return 0;
976
977 ret = fslmc_get_container_group(&groupid);
978 if (ret)
979 return ret;
980
981 /* In case this group was already opened, continue without any
982 * processing.
983 */
984 if (vfio_group.groupid == groupid) {
985 DPAA2_BUS_ERR("groupid already exists %d", groupid);
986 return 0;
987 }
988
989 /* Get the actual group fd */
990 ret = rte_vfio_get_group_fd(groupid);
991 if (ret < 0)
992 return ret;
993 vfio_group.fd = ret;
994
995 /* Check group viability */
996 ret = ioctl(vfio_group.fd, VFIO_GROUP_GET_STATUS, &status);
997 if (ret) {
998 DPAA2_BUS_ERR("VFIO error getting group status");
999 close(vfio_group.fd);
1000 rte_vfio_clear_group(vfio_group.fd);
1001 return ret;
1002 }
1003
1004 if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
1005 DPAA2_BUS_ERR("VFIO group not viable");
1006 close(vfio_group.fd);
1007 rte_vfio_clear_group(vfio_group.fd);
1008 return -EPERM;
1009 }
1010 /* Since Group is VIABLE, Store the groupid */
1011 vfio_group.groupid = groupid;
1012
1013 /* check if group does not have a container yet */
1014 if (!(status.flags & VFIO_GROUP_FLAGS_CONTAINER_SET)) {
1015 /* Now connect this IOMMU group to given container */
1016 ret = vfio_connect_container();
1017 if (ret) {
1018 DPAA2_BUS_ERR(
1019 "Error connecting container with groupid %d",
1020 groupid);
1021 close(vfio_group.fd);
1022 rte_vfio_clear_group(vfio_group.fd);
1023 return ret;
1024 }
1025 }
1026
1027 /* Get Device information */
1028 ret = ioctl(vfio_group.fd, VFIO_GROUP_GET_DEVICE_FD, fslmc_container);
1029 if (ret < 0) {
1030 DPAA2_BUS_ERR("Error getting device %s fd from group %d",
1031 fslmc_container, vfio_group.groupid);
1032 close(vfio_group.fd);
1033 rte_vfio_clear_group(vfio_group.fd);
1034 return ret;
1035 }
1036 container_device_fd = ret;
1037 DPAA2_BUS_DEBUG("VFIO Container FD is [0x%X]",
1038 container_device_fd);
1039
1040 return 0;
1041 }
1042